commit f182aa1e314b2105ebf789eba1b1b1f003ffd765 Author: Jonathan Cobb Date: Fri Dec 13 10:59:34 2019 -0500 first commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4c1ceb4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +*.iml +.idea +target +tmp +logs +dependency-reduced-pom.xml +velocity.log +*~ +build.log diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/bin/djbdns_rebuild.sh b/bin/djbdns_rebuild.sh new file mode 100644 index 0000000..4687d6b --- /dev/null +++ b/bin/djbdns_rebuild.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# +# Put this script in the same directory as your djbdns data file, usually /etc/tinydns/root +# +# Break up your monolithic data file into one-file per domain, and put them in the "domains" subdirectory. +# +# When this script runs, it concatenates all files in the "domains" subdirectory into a monolithic data +# file and tried to build it via make. If that fails, the old file is kept. +# +# Note that you may still want to restart the tinydns service (svc -h /path/to/tinydns) +# + +function die () { + echo "${1}" + exit 1 +} + +BASE=$(cd $(dirname $0) && pwd) + +if [ ! -f data ] ; then + echo "No data file found!" + exit 1 +fi + +if [ ! -d ${BASE}/domains ] ; then + echo "No domains dir found!" + exit 1 +fi + +mkdir -p ${BASE}/backups || die "Error creating backups dir" + +TODAY=$(date +%Y-%m-%d) +BACKUP=$(mktemp ${BASE}/backups/data.backup.${TODAY}.XXXXXXX) + +cp data ${BACKUP} || die "Error backing up data file" +CHANGED=$(mktemp data.tried.${TODAY}.XXXXXX) +for domain in $(find ${BASE}/domains -type f) ; do + echo "" >> ${CHANGED} + echo "####################" >> ${CHANGED} + echo "# $(basename ${domain})" >> ${CHANGED} + echo "####################" >> ${CHANGED} + cat ${domain} >> ${CHANGED} + echo "" >> ${CHANGED} +done + +mv ${CHANGED} data +make +if [ $? -ne 0 ] ; then + echo "Error rebuilding data file, rolling back. Failed data file is in ${CHANGED}" + mv data ${CHANGED} + mv ${BACKUP} data +fi diff --git a/bin/gsync b/bin/gsync new file mode 100755 index 0000000..e88065e --- /dev/null +++ b/bin/gsync @@ -0,0 +1,76 @@ +#!/bin/bash +# +# Usage: gsync [rsync-options] source destination +# +# Synchronize a git source directory with a remote directory, excluding files according to then +# rules found in .gitignore. If subdirectories also contain .gitignore files, then those rules +# will be applied (but only in each respective subdirectory). +# +# Note that ONLY .gitignore files in the directory where this command is run from will be considered +# Thus, when the source or destination is a local path, it should be specified relative to the current +# directory. +# +# There will be two rsync statements - one to exclude everything that should be excluded, +# and a second to handle the exceptions to the exclusion rules - the lines in .gitignore that begin with ! +# +# The exceptions to the exclusions are rsync'd first, and if that succeeds, the second rsync +# copies everything else. +# +# +# --- SUPPORT OPEN SOURCE --- +# If you find this script has saved you a decent amount time, please consider dropping me some coin. +# I will be forever grateful and your name will be permanently emblazoned on my Wall of Honor. +# My bitcoin wallet address is 1HoiSHKxYM4EtsP3xFGsY2xWYvh4hAuJ2q +# Paypal or Dwolla: jonathan (replace this with the 'AT' sign on your keyboard) kyuss.org +# +# Thank You. +# +# - jonathan. +# + +if [[ -z "${1}" || -z "${2}" || "${1}" == "--help" || "${1}" == "-help" || "${1}" == "-h" ]] ; then + echo "Usage: gsync [rsync-options] source destination" + exit 1 +fi + +includes="" +excludes='--exclude=.git*' +base="$(pwd)" + +function process_git_ignore () { + + git_ignore="${1}" + if [ "$(dirname ${git_ignore})" = "${base}" ] ; then + prefix="" + else + prefix=".$(echo -n "$(dirname ${git_ignore})" | sed -e 's,^'${base}',,')" + fi + + while read -r line || [[ -n "${line}" ]] ; do + # todo: there is probably a cleaner test for "first char == !" + if [ $(echo "${line}" | head -c 1 | grep -- '!' | wc -l) -gt 0 ] ; then + includes="${includes} + --include='${prefix}$(echo "${line}" | sed -e 's/^!//' | sed -e 's/ /\\ /g')'" + else + excludes="${excludes} + --exclude='${prefix}$(echo "${line}" | sed -e 's/ /\\ /g')'" + fi + done < ${git_ignore} + +} + +# root .gitignore file +if [ -f .gitignore ] ; then + process_git_ignore "$(pwd)/.gitignore" +fi + +# check for other .gitignore files +for i in $(find $(pwd) -mindepth 2 -type f -name .gitignore) ; do + process_git_ignore "${i}" +done + +rsync ${includes} --exclude="*" ${@} && rsync ${excludes} ${@} + +# for debugging +#echo "rsync ${includes} --exclude=\"*\" ${@}" && echo "rsync ${excludes} ${@}" +#echo "rsync ${excludes} ${@}" diff --git a/bin/json-editor b/bin/json-editor new file mode 100755 index 0000000..ad4c014 --- /dev/null +++ b/bin/json-editor @@ -0,0 +1,63 @@ +#!/usr/bin/python + +import sys +import json +import re + +from optparse import OptionParser + +parser = OptionParser() +parser.add_option("-i", "--infile", dest="infile", + help="read JSON from here (stdin if omitted)", metavar="FILE") +parser.add_option("-o", "--outfile", dest="outfile", + help="write JSON to here (stdout if omitted)", metavar="FILE") +parser.add_option("-p", "--path", dest="json_path", + help="JSON path to read", metavar="path.to.data") +parser.add_option("-v", "--value", dest="json_value", + help="Value to write to the JSON path", metavar="value") + +(options, args) = parser.parse_args() + +if options.infile is None: + data = json.load(sys.stdin) +else: + with open(options.infile) as infile: + data = json.load(infile) + +ref = data +if options.json_value is None: + # READ mode + if options.json_path is not None: + for token in re.split('\.', options.json_path): + try: + ref = ref[token] + except KeyError: + # JSON path does not exist, we treat that as "empty" + ref = None + break + data = ref + +else: + # WRITE mode + if options.json_path is not None: + token_path = re.split('\.', options.json_path) + if len(token_path) == 0: + data = options.json_value + else: + for token in token_path[0:-1]: + try: + ref = ref[token] + except KeyError: + # JSON path does not exist, create it + ref[token] = {} + ref = ref[token] + ref[token_path[-1]] = options.json_value + else: + data = options.json_value + +if ref is not None: + if options.outfile is None: + print json.dumps(data, sys.stdout, indent=2) + else: + with open(options.outfile, 'w') as outfile: + outfile.write(json.dumps(data, indent=2)) diff --git a/bin/log-classes.sh b/bin/log-classes.sh new file mode 100755 index 0000000..69292b9 --- /dev/null +++ b/bin/log-classes.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +grep class,load | awk '{print $2}' | awk -F '$' '{print $1}' | sort | uniq diff --git a/bin/log-packages.sh b/bin/log-packages.sh new file mode 100755 index 0000000..7928f38 --- /dev/null +++ b/bin/log-packages.sh @@ -0,0 +1,2 @@ +#!/bin/bash +grep class,load | awk '{print $2}' | awk -F '$' '{print $1}' | awk -F '.' 'BEGIN {OFS="."} {$(NF--)=""; print}' | sed -e 's/\.$//' | sort | uniq diff --git a/bin/minify-jre b/bin/minify-jre new file mode 100755 index 0000000..2479162 --- /dev/null +++ b/bin/minify-jre @@ -0,0 +1,18 @@ +#!/bin/bash + +function die () { + echo >&2 "${1}" + exit 1 +} + +if [[ -z "${JAVA_HOME}" ]] ; then + die "No JAVA_HOME env var defined" +fi + +OUTPUT_JRE=${1:?no output JRE dir provided} +if [[ -e ${OUTPUT_JRE} ]] ; then + die "Output JRE dir already exists: ${OUTPUT_JRE}" +fi +MODS=${2:-java.base} + +${JAVA_HOME}/bin/jlink --module-path ${JAVA_HOME}/jmods --add-modules ${MODS} --output ${OUTPUT_JRE} diff --git a/bin/patchjar b/bin/patchjar new file mode 100755 index 0000000..8c3e3fe --- /dev/null +++ b/bin/patchjar @@ -0,0 +1,42 @@ +#!/bin/bash +# +# patchjar - for quick patching jar files in a world of maven projects +# +# Usage: patchjar [ ...] +# +# jar-file: path to a jar file. you can use relative paths +# destination: a local path or user@host:path +# dirs: any directory with a maven pom.xml file +# +# Rationale: building uber-jars is time-consuming. Uber-jars are wonderful for +# deployment, but can sometimes feel like the slow down iteration cycles. +# +# patchjar runs "mvn compile" in each maven directory, then directly updates +# the jar via "jar uvf". Only the class files that changed are updated. +# +# Once all patches are applied, patchjar copies the jar file to the destination. +# + +function die () { + echo >&2 "${1}" + exit 1 +} + +START_DIR=$(pwd) +JAR="$(cd $(dirname ${1}) && pwd)/$(basename ${1})" +DESTINATION="${2}" +shift 2 + +MAVEN="mvn -DskipTests=true -Dcheckstyle.skip=true" + +for dir in ${@} ; do + cd ${START_DIR} && cd $(cd ${dir} && pwd) \ + && ${MAVEN} compile \ + && cd target/classes \ + && classes="$(find . -type f -mmin -3)" && if [ -z "${classes}" ] ; then classes="./*" ; fi \ + && jar uvf ${JAR} ${classes} \ + || die "Error building: ${dir}" +done + +cd ${START_DIR} +scp ${JAR} ${DESTINATION} diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..b13c831 --- /dev/null +++ b/pom.xml @@ -0,0 +1,368 @@ + + + + + 4.0.0 + + + org.cobbzilla + cobbzilla-parent + 1.0.0-SNAPSHOT + + + cobbzilla-utils + cobbzilla-utils + 1.0.0-SNAPSHOT + jar + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + uberjar + + + + org.apache.maven.plugins + maven-shade-plugin + 2.1 + + zilla-utils + + + + package + + shade + + + + + org.cobbzilla.util.main.IndexMain + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + + + + + org.graalvm.js + js + 19.2.0 + + + + org.graalvm.js + js-scriptengine + 19.2.0 + + + + + org.graalvm.truffle + truffle-api + 19.2.0 + + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + ${jackson.version} + + + + commons-beanutils + commons-beanutils + ${commons-beanutils.version} + + + + org.apache.commons + commons-collections4 + ${commons-collections.version} + + + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + + + + + org.apache.commons + commons-text + 1.7 + + + + org.apache.ant + ant + ${ant.version} + + + + commons-io + commons-io + ${commons-io.version} + + + + org.apache.commons + commons-compress + ${commons-compress.version} + + + + org.apache.commons + commons-exec + ${commons-exec.version} + + + + org.apache.httpcomponents + httpcore + ${httpcore.version} + + + org.apache.httpcomponents + httpclient + ${httpclient.version} + + + org.apache.httpcomponents + httpmime + ${httpmime.version} + + + + com.google.guava + guava + ${guava.version} + + + + joda-time + joda-time + ${joda-time.version} + + + + com.github.jknack + handlebars + ${handlebars.version} + + + com.github.jknack + handlebars-jackson2 + ${handlebars.version} + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.slf4j + jul-to-slf4j + ${slf4j.version} + + + ch.qos.logback + logback-classic + ${logback.version} + + + + org.quartz-scheduler + quartz + ${quartz.version} + + + c3p0 + c3p0 + + + + + org.quartz-scheduler + quartz-jobs + ${quartz.version} + + + + + org.projectlombok + lombok + ${lombok.version} + compile + + + + + + + + + + + jtidy + jtidy + ${jtidy.version} + + + xalan + xalan + ${xalan.version} + + + net.sf.saxon + Saxon-HE + 9.7.0-10 + + + + args4j + args4j + ${args4j.version} + + + + net.java.dev.jna + jna + 4.1.0 + + + + + com.codeborne + phantomjsdriver + + + 1.3.0 + + + + + fr.opensagres.xdocreport + fr.opensagres.xdocreport.document + ${xdocreport.version} + + + fr.opensagres.xdocreport + fr.opensagres.xdocreport.template.velocity + ${xdocreport.version} + + + fr.opensagres.xdocreport + fr.opensagres.xdocreport.document.docx + ${xdocreport.version} + + + fr.opensagres.xdocreport + fr.opensagres.xdocreport.converter.docx.xwpf + ${xdocreport.version} + + + fr.opensagres.xdocreport + org.apache.poi.xwpf.converter.pdf + + + + + fr.opensagres.xdocreport + org.apache.poi.xwpf.converter.pdf.itext5 + ${xdocreport.version} + + + + + org.apache.pdfbox + pdfbox + 2.0.16 + + + + + org.atteo + xml-combiner + 2.2 + + + + + io.github.bonigarcia + webdrivermanager + 2.1.0 + + + + + org.bouncycastle + bcprov-jdk15on + 1.64 + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 2.3.2 + + 11 + 11 + true + + + + + + diff --git a/src/main/java/org/cobbzilla/util/bean/BeanMerger.java b/src/main/java/org/cobbzilla/util/bean/BeanMerger.java new file mode 100644 index 0000000..37aa8cf --- /dev/null +++ b/src/main/java/org/cobbzilla/util/bean/BeanMerger.java @@ -0,0 +1,59 @@ +package org.cobbzilla.util.bean; + +import org.apache.commons.beanutils.PropertyUtilsBean; + +import java.beans.PropertyDescriptor; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; + +public class BeanMerger { + + private static final PropertyUtilsBean propertyUtils = new PropertyUtilsBean(); + + public static void mergeProperties(Object dest, Object orig) { + merge(dest, orig, AlwaysCopy.INSTANCE); + } + + public static void mergeNotNullProperties(Object dest, Object orig) { + merge(dest, orig, NotNull.INSTANCE); + } + + private static void merge(Object dest, Object orig, CopyEvaluator evaluator) { + + if (dest == null) throw new IllegalArgumentException ("No destination bean specified"); + if (orig == null) throw new IllegalArgumentException("No origin bean specified"); + + PropertyDescriptor[] origDescriptors = propertyUtils.getPropertyDescriptors(orig); + for (PropertyDescriptor origDescriptor : origDescriptors) { + String name = origDescriptor.getName(); + if ("class".equals(name)) { + continue; // No point in trying to set an object's class + } + if (propertyUtils.isReadable(orig, name) && + propertyUtils.isWriteable(dest, name)) { + try { + Object value = propertyUtils.getSimpleProperty(orig, name); + if (evaluator.shouldCopy(name, value)) { + propertyUtils.setProperty(dest, name, value); + } + } catch (NoSuchMethodException e) { + // Should not happen + } catch (Exception e) { + die("Error copying properties: " + e, e); + } + } + } + } + + private interface CopyEvaluator { + boolean shouldCopy(String name, Object value); + } + static class AlwaysCopy implements CopyEvaluator { + static final AlwaysCopy INSTANCE = new AlwaysCopy(); + @Override public boolean shouldCopy(String name, Object value) { return true; } + } + static class NotNull implements CopyEvaluator { + static final NotNull INSTANCE = new NotNull(); + @Override public boolean shouldCopy(String name, Object value) { return value != null; } + } +} diff --git a/src/main/java/org/cobbzilla/util/cache/AutoRefreshingReference.java b/src/main/java/org/cobbzilla/util/cache/AutoRefreshingReference.java new file mode 100644 index 0000000..612e2ea --- /dev/null +++ b/src/main/java/org/cobbzilla/util/cache/AutoRefreshingReference.java @@ -0,0 +1,43 @@ +package org.cobbzilla.util.cache; + +import lombok.Getter; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import static org.cobbzilla.util.daemon.ZillaRuntime.now; + +public abstract class AutoRefreshingReference { + + @Getter private final AtomicReference object = new AtomicReference<>(); + @Getter private final AtomicLong lastSet = new AtomicLong(); + + public abstract T refresh(); + public abstract long getTimeout(); + + public T get() { + synchronized (object) { + if (isEmpty() || now() - lastSet.get() > getTimeout()) update(); + return object.get(); + } + } + + public boolean isEmpty() { synchronized (object) { return object.get() == null; } } + + public void update() { + synchronized (object) { + object.set(refresh()); + lastSet.set(now()); + } + } + + public void flush() { set(null); } + + public void set(T thing) { + synchronized (object) { + object.set(thing); + lastSet.set(now()); + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/cache/BackgroundRefreshingReference.java b/src/main/java/org/cobbzilla/util/cache/BackgroundRefreshingReference.java new file mode 100644 index 0000000..f1c8b54 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/cache/BackgroundRefreshingReference.java @@ -0,0 +1,52 @@ +package org.cobbzilla.util.cache; + +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.system.Sleep; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public abstract class BackgroundRefreshingReference extends AutoRefreshingReference { + + private final AtomicBoolean updateInProgress = new AtomicBoolean(false); + private final Refresher refresher = new Refresher(); + private final AtomicInteger errorCount = new AtomicInteger(0); + + public boolean initialize () { return true; } + + public BackgroundRefreshingReference() { + if (initialize()) update(); + } + + @Override public void update() { + synchronized (updateInProgress) { + if (updateInProgress.get()) return; + updateInProgress.set(true); + new Thread(refresher).start(); + } + } + + private class Refresher implements Runnable { + @Override public void run() { + try { + int errCount = errorCount.get(); + if (errCount > 0) { + Sleep.sleep(TimeUnit.SECONDS.toMillis(1) * (long) Math.pow(2, Math.min(errCount, 6))); + } + set(refresh()); + errorCount.set(0); + + } catch (Exception e) { + log.warn("error refreshing: "+e); + errorCount.incrementAndGet(); + + } finally { + synchronized (updateInProgress) { + updateInProgress.set(false); + } + } + } + } +} diff --git a/src/main/java/org/cobbzilla/util/chef/VendorDatabag.java b/src/main/java/org/cobbzilla/util/chef/VendorDatabag.java new file mode 100644 index 0000000..965ecb1 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/chef/VendorDatabag.java @@ -0,0 +1,39 @@ +package org.cobbzilla.util.chef; + +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; +import org.cobbzilla.util.security.ShaUtil; + +import java.util.ArrayList; +import java.util.List; + +@Accessors(chain=true) +public class VendorDatabag { + + public static final VendorDatabag NULL = new VendorDatabag(); + + @Getter @Setter private String service_key_endpoint; + @Getter @Setter private String ssl_key_sha; + @Getter @Setter private List settings = new ArrayList<>(); + + public VendorDatabag addSetting (VendorDatabagSetting setting) { settings.add(setting); return this; } + + public VendorDatabagSetting getSetting(String path) { + for (VendorDatabagSetting s : settings) { + if (s.getPath().equals(path)) return s; + } + return null; + } + + public boolean containsSetting (String path) { return getSetting(path) != null; } + + public boolean isDefault (String path, String value) { + final VendorDatabagSetting setting = getSetting(path); + if (setting == null) return false; + + final String shasum = setting.getShasum(); + return shasum != null && ShaUtil.sha256_hex(value).equals(shasum); + + } +} diff --git a/src/main/java/org/cobbzilla/util/chef/VendorDatabagSetting.java b/src/main/java/org/cobbzilla/util/chef/VendorDatabagSetting.java new file mode 100644 index 0000000..9bd9611 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/chef/VendorDatabagSetting.java @@ -0,0 +1,20 @@ +package org.cobbzilla.util.chef; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@NoArgsConstructor @AllArgsConstructor +public class VendorDatabagSetting { + + @Getter @Setter private String path; + @Getter @Setter private String shasum; + @Getter @Setter private boolean block_ssh = false; + + public VendorDatabagSetting(String path, String shasum) { + setPath(path); + setShasum(shasum); + } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/AllowNullComparator.java b/src/main/java/org/cobbzilla/util/collection/AllowNullComparator.java new file mode 100644 index 0000000..f0cd5af --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/AllowNullComparator.java @@ -0,0 +1,20 @@ +package org.cobbzilla.util.collection; + +import java.util.Comparator; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; + +public class AllowNullComparator implements Comparator { + + public static final AllowNullComparator STRING = new AllowNullComparator<>(); + public static final AllowNullComparator INT = new AllowNullComparator<>(); + public static final AllowNullComparator LONG = new AllowNullComparator<>(); + + @Override public int compare(E o1, E o2) { + if (o1 == null) return o2 == null ? 0 : -1; + if (o2 == null) return 1; + if (o1 instanceof Comparable && o2 instanceof Comparable) return ((Comparable) o1).compareTo(o2); + return die("compare: incomparable objects: "+o1+", "+o2); + } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/ArrayUtil.java b/src/main/java/org/cobbzilla/util/collection/ArrayUtil.java new file mode 100644 index 0000000..ddb9939 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/ArrayUtil.java @@ -0,0 +1,145 @@ +package org.cobbzilla.util.collection; + +import org.cobbzilla.util.string.StringUtil; + +import java.lang.reflect.Array; +import java.util.*; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.reflect.ReflectionUtil.arrayClass; + +public class ArrayUtil { + + public static final Object[] SINGLE_NULL_OBJECT = new Object[]{null}; + public static final Object[] EMPTY_OBJECT_ARRAY = new Object[0]; + + public static T[] append (T[] array, T... elements) { + if (array == null || array.length == 0) { + if (elements.length == 0) return (T[]) new Object[]{}; // punt, it's empty anyway + final T[] newArray = (T[]) Array.newInstance(elements[0].getClass(), elements.length); + System.arraycopy(elements, 0, newArray, 0, elements.length); + return newArray; + } else { + if (elements.length == 0) return Arrays.copyOf(array, array.length); + final T[] copy = Arrays.copyOf(array, array.length + elements.length); + System.arraycopy(elements, 0, copy, array.length, elements.length); + return copy; + } + } + + public static T[] concat (T[]... arrays) { + int size = 0; + for (T[] array : arrays) { + size += array == null ? 0 : array.length; + } + final Class componentType = arrays.getClass().getComponentType().getComponentType(); + final T[] newArray = (T[]) Array.newInstance(componentType, size); + int destPos = 0; + for (T[] array : arrays) { + System.arraycopy(array, 0, newArray, destPos, array.length); + destPos += array.length; + } + return newArray; + } + + public static T[] remove(T[] array, int indexToRemove) { + if (array == null) throw new NullPointerException("remove: array was null"); + if (indexToRemove >= array.length || indexToRemove < 0) throw new IndexOutOfBoundsException("remove: cannot remove element "+indexToRemove+" from array of length "+array.length); + final List list = new ArrayList<>(Arrays.asList(array)); + list.remove(indexToRemove); + final T[] newArray = (T[]) Array.newInstance(array.getClass().getComponentType(), array.length-1); + return list.toArray(newArray); + } + + /** + * Return a slice of an array. If from == to then an empty array will be returned. + * @param array the source array + * @param from the start index, inclusive. If less than zero or greater than the length of the array, an Exception is thrown + * @param to the end index, NOT inclusive. If less than zero or greater than the length of the array, an Exception is thrown + * @param the of the array + * @return A slice of the array. The source array is not modified. + */ + public static T[] slice(T[] array, int from, int to) { + + if (array == null) throw new NullPointerException("slice: array was null"); + if (from < 0 || from > array.length) die("slice: invalid 'from' index ("+from+") for array of size "+array.length); + if (to < 0 || to < from || to > array.length) die("slice: invalid 'to' index ("+to+") for array of size "+array.length); + + final T[] newArray = (T[]) Array.newInstance(array.getClass().getComponentType(), to-from); + if (to == from) return newArray; + System.arraycopy(array, from, newArray, 0, to-from); + return newArray; + } + + public static List merge(Collection... collections) { + if (empty(collections)) return Collections.emptyList(); + final Set result = new HashSet<>(); + for (Collection c : collections) result.addAll(c); + return new ArrayList<>(result); + } + + /** + * Produce a delimited string from an array. Null values will appear as "null" + * @param array the array to consider + * @param delim the delimiter to put in between each element + * @return the result of calling .toString on each array element, or "null" for null elements, separated by the given delimiter. + */ + public static String arrayToString(Object[] array, String delim) { + return arrayToString(array, delim, "null"); + } + + /** + * Produce a delimited string from an array. + * @param array the array to consider + * @param delim the delimiter to put in between each element + * @param nullValue the value to write if an array entry is null. if this parameter is null, then null array entries will not be included in the output. + * @return a string that starts with [ and ends with ] and within is the result of calling .toString on each non-null element (and printing nullValue for each null element, unless nulValue == null in which case null elements are omitted), with 'delim' in between each entry. + */ + public static String arrayToString(Object[] array, String delim, String nullValue) { + return arrayToString(array, delim, nullValue, true); + } + + /** + * Produce a delimited string from an array. + * @param array the array to consider + * @param delim the delimiter to put in between each element + * @param nullValue the value to write if an array entry is null. if this parameter is null, then null array entries will not be included in the output. + * @param includeBrackets if false, the return value will not start/end with [] + * @return a string that starts with [ and ends with ] and within is the result of calling .toString on each non-null element (and printing nullValue for each null element, unless nulValue == null in which case null elements are omitted), with 'delim' in between each entry. + */ + public static String arrayToString(Object[] array, String delim, String nullValue, boolean includeBrackets) { + if (array == null) return "null"; + final StringBuilder b = new StringBuilder(); + for (Object o : array) { + if (b.length() > 0) b.append(delim); + if (o == null) { + if (nullValue == null) continue; + b.append(nullValue); + } else if (o.getClass().isArray()) { + b.append(arrayToString((Object[]) o, delim, nullValue)); + } else if (o instanceof Map) { + b.append(StringUtil.toString((Map) o)); + } else { + b.append(o.toString()); + } + } + return includeBrackets ? b.insert(0, "[").append("]").toString() : b.toString(); + } + + public static T[] shift(T[] args) { + if (args == null) return null; + if (args.length == 0) return args; + final T[] newArgs = (T[]) Array.newInstance(args[0].getClass(), args.length-1); + System.arraycopy(args, 1, newArgs, 0, args.length-1); + return newArgs; + } + + public static T[] singletonArray (T thing) { return singletonArray(thing, (Class) thing.getClass()); } + + public static T[] singletonArray (T thing, Class clazz) { + final T[] array = (T[]) Array.newInstance(arrayClass(clazz), 1); + array[0] = thing; + return array; + } +} diff --git a/src/main/java/org/cobbzilla/util/collection/CaseInsensitiveStringKeyMap.java b/src/main/java/org/cobbzilla/util/collection/CaseInsensitiveStringKeyMap.java new file mode 100644 index 0000000..9b90264 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/CaseInsensitiveStringKeyMap.java @@ -0,0 +1,55 @@ +package org.cobbzilla.util.collection; + +import java.util.Collections; +import java.util.Enumeration; +import java.util.concurrent.ConcurrentHashMap; + +public class CaseInsensitiveStringKeyMap extends ConcurrentHashMap { + + private ConcurrentHashMap origKeys = new ConcurrentHashMap<>(); + + public String key(Object key) { return key == null ? null : key.toString().toLowerCase(); } + + @Override public KeySetView keySet() { return super.keySet(); } + + @Override public Enumeration keys() { return Collections.enumeration(origKeys.values()); } + + @Override public V get(Object key) { return super.get(key(key)); } + + @Override public boolean containsKey(Object key) { return super.containsKey(key(key)); } + + @Override public V put(String key, V value) { + final String ciKey = key(key); + origKeys.put(ciKey, key); + return super.put(ciKey, value); + } + + @Override public V putIfAbsent(String key, V value) { + final String ciKey = key(key); + origKeys.putIfAbsent(ciKey, key); + return super.putIfAbsent(ciKey, value); + } + + @Override public V remove(Object key) { + final String ciKey = key(key); + origKeys.remove(ciKey); + return super.remove(ciKey); + } + + @Override public boolean remove(Object key, Object value) { + final String ciKey = key(key); + origKeys.remove(ciKey, value); + return super.remove(ciKey, value); + } + + @Override public boolean replace(String key, V oldValue, V newValue) { + final String ciKey = key(key); + return super.replace(ciKey, oldValue, newValue); + } + + @Override public V replace(String key, V value) { + final String ciKey = key(key); + return super.replace(ciKey, value); + } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/CaseInsensitiveStringSet.java b/src/main/java/org/cobbzilla/util/collection/CaseInsensitiveStringSet.java new file mode 100644 index 0000000..9be888a --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/CaseInsensitiveStringSet.java @@ -0,0 +1,12 @@ +package org.cobbzilla.util.collection; + +import java.util.Collection; +import java.util.TreeSet; + +public class CaseInsensitiveStringSet extends TreeSet { + + public CaseInsensitiveStringSet() { super(String.CASE_INSENSITIVE_ORDER); } + + public CaseInsensitiveStringSet(Collection c) { addAll(c); } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/CollectionSource.java b/src/main/java/org/cobbzilla/util/collection/CollectionSource.java new file mode 100644 index 0000000..ff2fea2 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/CollectionSource.java @@ -0,0 +1,12 @@ +package org.cobbzilla.util.collection; + +import java.util.Collection; + +public interface CollectionSource { + + void addValue (T val); + void addValues (Collection vals); + + Collection getValues(); + +} diff --git a/src/main/java/org/cobbzilla/util/collection/CombinationsGenerator.java b/src/main/java/org/cobbzilla/util/collection/CombinationsGenerator.java new file mode 100644 index 0000000..0f01796 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/CombinationsGenerator.java @@ -0,0 +1,55 @@ +package org.cobbzilla.util.collection; + +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Set; + +public class CombinationsGenerator { + + // The main function that gets all combinations of size n-1 to 1, in set of size n. + // This function mainly uses combinationUtil() + public static Set> generateCombinations(Set elements) { + Set> result = new LinkedHashSet<>(); + + // i - A number of elements which will be used in the combination in this iteration + for (int i = elements.size() - 1; i >= 1 ; i--) { + // A temporary array to store all combinations one by one + String[] data = new String[i]; + // Get all combination using temporary array 'data' + result = _generate(result, elements.toArray(new String[elements.size()]), + data, 0, elements.size() - 1, 0, i); + } + return result; + } + + /** + * @param combinations - Resulting array with all combinations of arr + * @param arr - Input Array + * @param data - Temporary array to store current combination + * @param start - Staring index in arr[] + * @param end - Ending index in arr[] + * @param index - Current index in data[] + * @param r - Size of a combination + */ + private static Set> _generate(Set> combinations, String[] arr, + String[] data, int start, int end, int index, int r) { + // Current combination is ready + if (index == r) { + Set current = new HashSet<>(); + for (int j = 0; j < r; j++) { + current.add(data[j]); + } + combinations.add(current); + return combinations; + } + + // replace index with all possible elements. The condition `end - i + 1 >= r - index` makes sure that including + // one element at index will make a combination with remaining elements at remaining positions + for (int i = start; i <= end && end - i + 1 >= r - index; i++) { + data[index] = arr[i]; + combinations = _generate(combinations, arr, data, i + 1, end, index + 1, r); + } + + return combinations; + } +} diff --git a/src/main/java/org/cobbzilla/util/collection/ComparisonOperator.java b/src/main/java/org/cobbzilla/util/collection/ComparisonOperator.java new file mode 100644 index 0000000..d84b775 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/ComparisonOperator.java @@ -0,0 +1,23 @@ +package org.cobbzilla.util.collection; + +import com.fasterxml.jackson.annotation.JsonCreator; +import lombok.AllArgsConstructor; +import lombok.Getter; + +@AllArgsConstructor +public enum ComparisonOperator { + + lt ("<", "<", "-lt"), + le ("<=", "<=", "-le"), + eq ("=", "==", "-eq"), + ge (">=", ">=", "-ge"), + gt (">", ">", "-gt"), + ne ("!=", "!=", "-ne"); + + @Getter public final String sql; + @Getter public final String java; + @Getter public final String shell; + + @JsonCreator public static ComparisonOperator fromString(String val) { return valueOf(val.toLowerCase()); } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/CustomHashSet.java b/src/main/java/org/cobbzilla/util/collection/CustomHashSet.java new file mode 100644 index 0000000..3457f29 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/CustomHashSet.java @@ -0,0 +1,94 @@ +package org.cobbzilla.util.collection; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +@NoArgsConstructor @Accessors(chain=true) +public class CustomHashSet implements Set { + + public interface Hasher { String hash (E thing); } + + @Getter @Setter private Class elementClass; + @Getter @Setter private Hasher hasher; + + private Map map = new ConcurrentHashMap<>(); + + public CustomHashSet(Class clazz, Hasher hasher, Collection collection) { + this(clazz, hasher); + addAll(collection); + } + + public CustomHashSet(Class elementClass, Hasher hasher) { + this.elementClass = elementClass; + this.hasher = hasher; + } + + @Override public int size() { return map.size(); } + + @Override public boolean isEmpty() { return map.isEmpty(); } + + @Override public boolean contains(Object o) { + if (o == null) return false; + if (getElementClass().isAssignableFrom(o.getClass())) { + return map.containsKey(hasher.hash(o)); + + } else if (o instanceof String) { + return map.containsKey(o); + } + return false; + } + + @Override public Iterator iterator() { return map.values().iterator(); } + + @Override public Object[] toArray() { return map.values().toArray(); } + + @Override public T[] toArray(T[] a) { return (T[]) map.values().toArray(); } + + @Override public boolean add(E e) { return map.put(hasher.hash(e), e) == null; } + + public E find(E e) { return map.get(hasher.hash(e)); } + + @Override public boolean remove(Object o) { + if (getElementClass().isAssignableFrom(o.getClass())) { + return map.remove(hasher.hash(o)) != null; + + } else if (o instanceof String) { + return map.remove(o) != null; + } + return false; + } + + @Override public boolean containsAll(Collection c) { + for (Object o : c) if (!contains(o)) return false; + return true; + } + + @Override public boolean addAll(Collection c) { + boolean anyAdded = false; + for (E o : c) if (!add(o)) anyAdded = true; + return anyAdded; + } + + @Override public boolean retainAll(Collection c) { + final Set toRemove = new HashSet<>(); + for (Map.Entry entry : map.entrySet()) { + if (!c.contains(entry.getValue())) toRemove.add(entry.getKey()); + } + for (String k : toRemove) remove(k); + return !toRemove.isEmpty(); + } + + @Override public boolean removeAll(Collection c) { + boolean anyRemoved = false; + for (Object o : c) if (map.remove(o) != null) anyRemoved = true; + return anyRemoved; + } + + @Override public void clear() { map.clear(); } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/Expandable.java b/src/main/java/org/cobbzilla/util/collection/Expandable.java new file mode 100644 index 0000000..9a54c84 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/Expandable.java @@ -0,0 +1,10 @@ +package org.cobbzilla.util.collection; + +import java.util.List; +import java.util.Map; + +public interface Expandable { + + List expand(Map context); + +} diff --git a/src/main/java/org/cobbzilla/util/collection/ExpirationMap.java b/src/main/java/org/cobbzilla/util/collection/ExpirationMap.java new file mode 100644 index 0000000..2f8f7a7 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/ExpirationMap.java @@ -0,0 +1,130 @@ +package org.cobbzilla.util.collection; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.experimental.Accessors; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported; +import static org.cobbzilla.util.daemon.ZillaRuntime.now; + +public class ExpirationMap implements Map { + + private final Map> map; + private long expiration = TimeUnit.HOURS.toMillis(1); + private long cleanInterval = TimeUnit.HOURS.toMillis(4); + private long lastCleaned = 0; + + public ExpirationMap() { + this.map = new ConcurrentHashMap<>(); + } + + public ExpirationMap(long expiration) { + this.map = new ConcurrentHashMap<>(); + this.expiration = expiration; + } + + public ExpirationMap(long expiration, long cleanInterval) { + this(expiration); + this.cleanInterval = cleanInterval; + } + + public ExpirationMap(long expiration, long cleanInterval, int initialCapacity) { + this.map = new ConcurrentHashMap<>(initialCapacity); + this.expiration = expiration; + this.cleanInterval = cleanInterval; + } + + public ExpirationMap(long expiration, long cleanInterval, int initialCapacity, float loadFactor) { + this.map = new ConcurrentHashMap<>(initialCapacity, loadFactor); + this.expiration = expiration; + this.cleanInterval = cleanInterval; + } + + public ExpirationMap(long expiration, long cleanInterval, int initialCapacity, float loadFactor, int concurrencyLevel) { + this.map = new ConcurrentHashMap<>(initialCapacity, loadFactor, concurrencyLevel); + this.expiration = expiration; + this.cleanInterval = cleanInterval; + } + + @Accessors(chain=true) + private class ExpirationMapEntry { + public final VAL value; + public volatile long atime = now(); + public ExpirationMapEntry(VAL value) { this.value = value; } + + public VAL touch() { atime = now(); return value; } + public boolean expired() { return now() > atime+expiration; } + } + + @Override public int size() { return map.size(); } + + @Override public boolean isEmpty() { return map.isEmpty(); } + + @Override public boolean containsKey(Object key) { return map.containsKey(key); } + + @Override public boolean containsValue(Object value) { + for (ExpirationMapEntry val : map.values()) { + if (val.value == value) return true; + } + return false; + } + + @Override public V get(Object key) { + final ExpirationMapEntry value = map.get(key); + return value == null ? null : value.touch(); + } + + @Override public V put(K key, V value) { + if (lastCleaned+cleanInterval > now()) cleanExpired(); + final ExpirationMapEntry previous = map.put(key, new ExpirationMapEntry<>(value)); + return previous == null ? null : previous.value; + } + + @Override public V remove(Object key) { + final ExpirationMapEntry previous = map.remove(key); + return previous == null ? null : previous.value; + } + + @Override public void putAll(Map m) { + for (Map.Entry entry : m.entrySet()) { + put(entry.getKey(), entry.getValue()); + } + } + + @Override public void clear() { map.clear(); } + + @Override public Set keySet() { return map.keySet(); } + + @Override public Collection values() { + return map.values().stream().map(v -> v.value).collect(Collectors.toList()); + } + + @AllArgsConstructor + private static class EMEntry implements Entry { + @Getter private K key; + @Getter private V value; + @Override public V setValue(V value) { return notSupported("setValue"); } + } + + @Override public Set> entrySet() { + return map.entrySet().stream().map(e -> new EMEntry<>(e.getKey(), e.getValue().value)).collect(Collectors.toSet()); + } + + private synchronized void cleanExpired () { + if (lastCleaned+cleanInterval < now()) return; + lastCleaned = now(); + final Set toRemove = new HashSet<>(); + for (Map.Entry> entry : map.entrySet()) { + if (entry.getValue().expired()) toRemove.add(entry.getKey()); + } + for (K k : toRemove) map.remove(k); + } +} diff --git a/src/main/java/org/cobbzilla/util/collection/FailedOperationCounter.java b/src/main/java/org/cobbzilla/util/collection/FailedOperationCounter.java new file mode 100644 index 0000000..16e91e3 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/FailedOperationCounter.java @@ -0,0 +1,47 @@ +package org.cobbzilla.util.collection; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; + +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import static org.cobbzilla.util.daemon.ZillaRuntime.now; + +@NoArgsConstructor @AllArgsConstructor @Accessors(chain=true) @Slf4j +public class FailedOperationCounter extends ConcurrentHashMap> { + + @Getter @Setter private long expiration = TimeUnit.MINUTES.toMillis(5); + @Getter @Setter private int maxFailures = 1; + + public void fail(T value) { + Map failures = get(value); + if (failures == null) { + failures = new ConcurrentHashMap<>(); + put(value, failures); + } + final long ftime = now(); + failures.put(ftime, ftime); + } + + public boolean tooManyFailures(T value) { + final Map failures = get(value); + if (failures == null) return false; + int count = 0; + for (Iterator iter = failures.keySet().iterator(); iter.hasNext();) { + Long ftime = iter.next(); + if (now() - ftime > expiration) { + iter.remove(); + } else { + if (++count >= maxFailures) return true; + } + } + return false; + } +} diff --git a/src/main/java/org/cobbzilla/util/collection/FieldTransformer.java b/src/main/java/org/cobbzilla/util/collection/FieldTransformer.java new file mode 100644 index 0000000..27f7850 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/FieldTransformer.java @@ -0,0 +1,39 @@ +package org.cobbzilla.util.collection; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.Transformer; +import org.cobbzilla.util.reflect.ReflectionUtil; + +import java.lang.reflect.Array; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.cobbzilla.util.collection.ArrayUtil.EMPTY_OBJECT_ARRAY; + +@AllArgsConstructor +public class FieldTransformer implements Transformer { + + public static final FieldTransformer TO_NAME = new FieldTransformer("name"); + public static final FieldTransformer TO_ID = new FieldTransformer("id"); + public static final FieldTransformer TO_UUID = new FieldTransformer("uuid"); + + @Getter private final String field; + + @Override public Object transform(Object o) { return ReflectionUtil.get(o, field); } + + public List collect (Collection c) { return c == null ? null : (List) CollectionUtils.collect(c, this); } + public Set collectSet (Collection c) { return c == null ? null : new HashSet<>(CollectionUtils.collect(c, this)); } + + public E[] array (Collection c) { + if (c == null) return null; + if (c.isEmpty()) return (E[]) EMPTY_OBJECT_ARRAY; + final List collect = (List) CollectionUtils.collect(c, this); + final Class elementType = (Class) ReflectionUtil.getterType(c.iterator().next(), field); + return collect.toArray((E[]) Array.newInstance(elementType, collect.size())); + } + +} \ No newline at end of file diff --git a/src/main/java/org/cobbzilla/util/collection/HasPriority.java b/src/main/java/org/cobbzilla/util/collection/HasPriority.java new file mode 100644 index 0000000..190e12c --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/HasPriority.java @@ -0,0 +1,35 @@ +package org.cobbzilla.util.collection; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; + +public interface HasPriority { + + Integer getPriority (); + + default boolean hasPriority () { return getPriority() != null; } + + Comparator SORT_PRIORITY = (r1, r2) -> { + if (!r2.hasPriority()) return r1.hasPriority() ? -1 : 0; + if (!r1.hasPriority()) return 1; + return r1.getPriority().compareTo(r2.getPriority()); + }; + + static int compare(Object o1, Object o2) { + return o1 instanceof HasPriority && o2 instanceof HasPriority ? SORT_PRIORITY.compare((HasPriority) o1, (HasPriority) o2) : 0; + } + + static Collection priorityAsc (Collection c) { + final Collection sorted = new ArrayList<>(c); + ((ArrayList) sorted).sort(SORT_PRIORITY); + return sorted; + } + + static Collection priorityDesc (Collection c) { + final Collection sorted = new ArrayList<>(c); + ((ArrayList) sorted).sort(SORT_PRIORITY.reversed()); + return sorted; + } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/InspectCollection.java b/src/main/java/org/cobbzilla/util/collection/InspectCollection.java new file mode 100644 index 0000000..c916a29 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/InspectCollection.java @@ -0,0 +1,39 @@ +package org.cobbzilla.util.collection; + +import java.util.*; + +public class InspectCollection { + + public static boolean containsCircularReference(String start, Map> graph) { + return containsCircularReference(new HashSet(), start, graph); + } + + public static boolean containsCircularReference(Set found, String start, Map> graph) { + final List descendents = graph.get(start); + if (descendents == null) return false; // special case: our starting point is outside the graph. + for (String target : descendents) { + if (found.contains(target)) { + // we've seen this target already, we have a circular reference + return true; + } + if (graph.containsKey(target)) { + // this target is also a member of the graph -- add to found and recurse + found.add(target); + if (containsCircularReference(new HashSet<>(found), target, graph)) return true; + } + // no "else" clause here: we don't care about anything not in the graph, it can't create a circular reference. + } + return false; + } + + public static boolean isLargerThan (Collection c, int size) { + int count = 0; + final Iterator i = c.iterator(); + while (i.hasNext() && count <= size) { + i.next(); + count++; + } + return count > size; + } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/ListUtil.java b/src/main/java/org/cobbzilla/util/collection/ListUtil.java new file mode 100644 index 0000000..6917dbc --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/ListUtil.java @@ -0,0 +1,78 @@ +package org.cobbzilla.util.collection; + +import com.google.common.collect.Lists; +import org.cobbzilla.util.reflect.ReflectionUtil; + +import java.util.*; + +public class ListUtil { + + public static List concat(List list1, List list2) { + if (list1 == null || list1.isEmpty()) return list2 == null ? null : new ArrayList<>(list2); + if (list2 == null || list2.isEmpty()) return new ArrayList<>(list1); + final List newList = new ArrayList<>(list1.size() + list2.size()); + newList.addAll(list1); + newList.addAll(list2); + return newList; + } + + // adapted from: https://stackoverflow.com/a/23870892/1251543 + + /** + * Combines several collections of elements and create permutations of all of them, taking one element from each + * collection, and keeping the same order in resultant lists as the one in original list of collections. + *

+ *

    Example + *
  • Input = { {a,b,c} , {1,2,3,4} }
    • + *
  • Output = { {a,1} , {a,2} , {a,3} , {a,4} , {b,1} , {b,2} , {b,3} , {b,4} , {c,1} , {c,2} , {c,3} , {c,4} }
    • + *
+ * + * @param collections Original list of collections which elements have to be combined. + * @return Resultant collection of lists with all permutations of original list. + */ + public static List> permutations(List> collections) { + if (collections == null || collections.isEmpty()) { + return Collections.emptyList(); + } else { + List> res = Lists.newLinkedList(); + permutationsImpl(collections, res, 0, new LinkedList()); + return res; + } + } + + private static void permutationsImpl(List> ori, Collection> res, int d, List current) { + // if depth equals number of original collections, final reached, add and return + if (d == ori.size()) { + res.add(current); + return; + } + + // iterate from current collection and copy 'current' element N times, one for each element + Collection currentCollection = ori.get(d); + for (T element : currentCollection) { + List copy = Lists.newLinkedList(current); + copy.add(element); + permutationsImpl(ori, res, d + 1, copy); + } + } + + public static List expand(Object[] things, Map context) { + final List results = new ArrayList<>(); + for (Object thing : things) { + if (thing instanceof Expandable) { + results.addAll(((Expandable) thing).expand(context)); + } else { + results.add(thing); + } + } + return results; + } + + public static List deepCopy(List list) { + if (list == null) return null; + final List copy = new ArrayList<>(); + for (T item : list) copy.add(item == null ? null : ReflectionUtil.copy(item)); + return copy; + } + +} \ No newline at end of file diff --git a/src/main/java/org/cobbzilla/util/collection/MapBuilder.java b/src/main/java/org/cobbzilla/util/collection/MapBuilder.java new file mode 100644 index 0000000..169b08d --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/MapBuilder.java @@ -0,0 +1,71 @@ +package org.cobbzilla.util.collection; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.cobbzilla.util.reflect.ReflectionUtil.instantiate; + +/** + * A handy utility for creating and initializing Maps in a single statement. + * @author Jonathan Cobb. + */ +public class MapBuilder { + + /** + * Most common create/init case. Usage: + * + * Map myPremadeMap = MapBuilder.build(new Object[][]{ + * { "a", true }, { "b", false }, { "c", true }, { "d", true }, + * { "e", "yes, still dangerous but at least it's not an anonymous class" } + * }); + * + * If your keys and values are of the same type, it will even be typesafe: + * Map someProperties = MapBuilder.build(new String[][]{ + * {"propA", "valueA" }, { "propB", "valueB" } + * }); + * + * @param values [x][2] array. items at [x][0] are keys and [x][1] are values. + * @return a LinkedHashMap (to preserve order of declaration) with the "values" mappings + */ + public static Map build(Object[][] values) { + return build((Map) new LinkedHashMap<>(), values); + } + + /** + * Usage: + * Map myMap = MapBuilder.build(new MyMapClass(options), + * new Object[][]{ {k,v}, {k,v}, ... }); + * @param map add key/value pairs to this map + * @return the map passed in, now containing new "values" mappings + */ + public static Map build(Map map, Object[][] values) { + for (Object[] value : values) { + map.put((K) value[0], (V) value[1]); + } + return map; + } + + /** Same as above, for single-value maps */ + public static Map build(Map map, K key, V value) { + return build(map, new Object[][]{{key,value}}); + } + + /** + * Usage: + * Map myMap = MapBuilder.build(MyMapClass.class, new Object[][]{ {k,v}, {k,v}, ... }); + * @param mapClass a Class that implements Map + * @return the map passed in, now containing new "values" mappings + */ + public static Map build(Class> mapClass, Object[][] values) { + return build(instantiate(mapClass), values); + } + + /** Usage: Map myMap = MapBuilder.build(key, value); */ + public static Map build(K key, V value) { + Map map = new HashMap<>(); + map.put(key, value); + return map; + } + +} \ No newline at end of file diff --git a/src/main/java/org/cobbzilla/util/collection/MapUtil.java b/src/main/java/org/cobbzilla/util/collection/MapUtil.java new file mode 100644 index 0000000..e68c3f2 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/MapUtil.java @@ -0,0 +1,42 @@ +package org.cobbzilla.util.collection; + +import com.fasterxml.jackson.core.type.TypeReference; + +import java.util.*; + +public class MapUtil { + + public static final TypeReference> JSON_STRING_OBJECT_MAP = new TypeReference>() {}; + public static final TypeReference> JSON_STRING_STRING_MAP = new TypeReference>() {}; + + public static Map toMap (Properties props) { + if (props == null || props.isEmpty()) return Collections.emptyMap(); + final Map map = new LinkedHashMap<>(props.size()); + for (String name : props.stringPropertyNames()) map.put(name, props.getProperty(name)); + return map; + } + + public static boolean deepEquals (Map m1, Map m2) { + if (m1 == null) return m2 == null; + if (m2 == null) return false; + if (m1.size() != m2.size()) return false; + final Set> set = m1.entrySet(); + for (Map.Entry e : set) { + V m1v = e.getValue(); + V m2v = m2.get(e.getKey()); + if (m2v == null) return false; + if ((m1v instanceof Map && !deepEquals((Map) m1v, (Map) m2v)) || !m1v.equals(m2v)) { + return false; + } + } + return true; + } + + public static int deepHash(Map m) { + int hash = 0; + for (Map.Entry e : m.entrySet()) { + hash = (31 * hash) + e.getKey().hashCode() + (31 * e.getValue().hashCode()); + } + return hash; + } +} diff --git a/src/main/java/org/cobbzilla/util/collection/NameAndValue.java b/src/main/java/org/cobbzilla/util/collection/NameAndValue.java new file mode 100644 index 0000000..adcf666 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/NameAndValue.java @@ -0,0 +1,102 @@ +package org.cobbzilla.util.collection; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; +import org.cobbzilla.util.javascript.JsEngine; + +import java.util.*; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +@NoArgsConstructor @AllArgsConstructor @Accessors(chain=true) +public class NameAndValue { + + public static final NameAndValue[] EMPTY_ARRAY = new NameAndValue[0]; + public static final Comparator NAME_COMPARATOR = Comparator.comparing(NameAndValue::getName); + + public static List map2list(Map map) { + final List list = new ArrayList<>(map.size()); + for (Map.Entry entry : map.entrySet()) { + list.add(new NameAndValue(entry.getKey(), entry.getValue() == null ? null : entry.getValue().toString())); + } + return list; + } + + @Getter @Setter private String name; + + public static Integer findInt(NameAndValue[] pairs, String name) { + final String val = find(pairs, name); + return val == null ? null : Integer.parseInt(val); + } + + public static String find(NameAndValue[] pairs, String name) { + if (pairs == null) return null; + for (NameAndValue pair : pairs) if (pair.getName().equals(name)) return pair.getValue(); + return null; + } + + public static String find(Collection pairs, String name) { + if (pairs == null || pairs.isEmpty()) return null; + return pairs.stream() + .filter(p -> p.getName().equals(name)) + .findFirst() + .map(NameAndValue::getValue) + .orElse(null); + } + + public static NameAndValue[] update(NameAndValue[] params, String name, String value) { + if (params == null) return new NameAndValue[] { new NameAndValue(name, value) }; + for (NameAndValue pair : params) { + if (pair.getName().equals(name)) { + pair.setValue(value); + return params; + } + } + return ArrayUtil.append(params, new NameAndValue(name, value)); + } + + public boolean hasName () { return !empty(name); } + @JsonIgnore public boolean getHasName () { return !empty(name); } + + @Getter @Setter private String value; + public boolean hasValue () { return !empty(value); } + @JsonIgnore public boolean getHasValue () { return !empty(value); } + + @Override public String toString() { return getName()+": "+getValue(); } + + public static NameAndValue[] evaluate (NameAndValue[] pairs, Map context) { + return evaluate(pairs, context, new JsEngine()); + } + + public static NameAndValue[] evaluate (NameAndValue[] pairs, Map context, JsEngine engine) { + + if (empty(context) || empty(pairs)) return pairs; + + final NameAndValue[] results = new NameAndValue[pairs.length]; + for (int i=0; i toMap(NameAndValue[] attrs) { + final Map map = new HashMap<>(); + if (!empty(attrs)) { + for (NameAndValue attr : attrs) { + map.put(attr.getName(), attr.value); + } + } + return map; + } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/NumberComparator.java b/src/main/java/org/cobbzilla/util/collection/NumberComparator.java new file mode 100644 index 0000000..91da5b7 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/NumberComparator.java @@ -0,0 +1,15 @@ +package org.cobbzilla.util.collection; + +import java.math.BigDecimal; +import java.util.Comparator; + +// adapted from: https://stackoverflow.com/a/2683388/1251543 +public class NumberComparator implements Comparator { + + public static final NumberComparator INSTANCE = new NumberComparator(); + + public int compare(Number a, Number b){ + return new BigDecimal(a.toString()).compareTo(new BigDecimal(b.toString())); + } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/SetSourceBase.java b/src/main/java/org/cobbzilla/util/collection/SetSourceBase.java new file mode 100644 index 0000000..60f57a9 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/SetSourceBase.java @@ -0,0 +1,28 @@ +package org.cobbzilla.util.collection; + +import lombok.ToString; +import org.cobbzilla.util.string.StringUtil; + +import java.util.Collection; +import java.util.HashSet; +import java.util.concurrent.atomic.AtomicReference; + +public class SetSourceBase implements CollectionSource { + + private final AtomicReference> values = new AtomicReference(new HashSet()); + + @Override public Collection getValues() { + synchronized (values) { return new HashSet<>(values.get()); } + } + + @Override public void addValue (T val) { + synchronized (values) { values.get().add(val); } + } + + @Override public void addValues (Collection vals) { + synchronized (values) { values.get().addAll(vals); } + } + + @Override public String toString () { return StringUtil.toString(values.get(), ", "); } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/SingletonList.java b/src/main/java/org/cobbzilla/util/collection/SingletonList.java new file mode 100644 index 0000000..14694f8 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/SingletonList.java @@ -0,0 +1,24 @@ +package org.cobbzilla.util.collection; + +import java.util.ArrayList; +import java.util.Collection; + +public class SingletonList extends ArrayList { + + public SingletonList (E element) { super.add(element); } + + @Override public E set(int index, E element) { throw unsupported(); } + @Override public boolean add(E e) { throw unsupported(); } + @Override public void add(int index, E element) { throw unsupported(); } + @Override public E remove(int index) { throw unsupported(); } + @Override public boolean remove(Object o) { throw unsupported(); } + @Override public void clear() { throw unsupported(); } + @Override public boolean addAll(Collection c) { throw unsupported(); } + @Override public boolean addAll(int index, Collection c) { throw unsupported(); } + @Override protected void removeRange(int fromIndex, int toIndex) { throw unsupported(); } + @Override public boolean removeAll(Collection c) { throw unsupported(); } + @Override public boolean retainAll(Collection c) { throw unsupported(); } + + private UnsupportedOperationException unsupported () { return new UnsupportedOperationException("singleton list is immutable"); } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/SingletonSet.java b/src/main/java/org/cobbzilla/util/collection/SingletonSet.java new file mode 100644 index 0000000..6c1b640 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/SingletonSet.java @@ -0,0 +1,18 @@ +package org.cobbzilla.util.collection; + +import java.util.Collection; +import java.util.HashSet; + +import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported; + +public class SingletonSet extends HashSet { + + public SingletonSet (E element) { super.add(element); } + + @Override public boolean add(E e) { return notSupported(); } + @Override public boolean remove(Object o) { return notSupported(); } + @Override public void clear() { notSupported(); } + @Override public boolean addAll(Collection c) { return notSupported(); } + @Override public boolean retainAll(Collection c) { return notSupported(); } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/Sorter.java b/src/main/java/org/cobbzilla/util/collection/Sorter.java new file mode 100644 index 0000000..dd74f1c --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/Sorter.java @@ -0,0 +1,18 @@ +package org.cobbzilla.util.collection; + +import java.util.Collection; +import java.util.Comparator; +import java.util.TreeSet; + +public class Sorter { + + public static Collection sort (Collection things, Comparator sorter) { + return sort(things, new TreeSet<>(sorter)); + } + + public static C sort (Collection things, C rval) { + rval.addAll(things); + return rval; + } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/StringPrefixTransformer.java b/src/main/java/org/cobbzilla/util/collection/StringPrefixTransformer.java new file mode 100644 index 0000000..30302cc --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/StringPrefixTransformer.java @@ -0,0 +1,14 @@ +package org.cobbzilla.util.collection; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.apache.commons.collections.Transformer; + +@AllArgsConstructor +public class StringPrefixTransformer implements Transformer { + + @Getter private String prefix; + + @Override public Object transform(Object input) { return prefix + input.toString(); } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/StringSetSource.java b/src/main/java/org/cobbzilla/util/collection/StringSetSource.java new file mode 100644 index 0000000..5b39f67 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/StringSetSource.java @@ -0,0 +1,3 @@ +package org.cobbzilla.util.collection; + +public class StringSetSource extends SetSourceBase {} diff --git a/src/main/java/org/cobbzilla/util/collection/ToStringTransformer.java b/src/main/java/org/cobbzilla/util/collection/ToStringTransformer.java new file mode 100644 index 0000000..e951fab --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/ToStringTransformer.java @@ -0,0 +1,11 @@ +package org.cobbzilla.util.collection; + +import org.apache.commons.collections.Transformer; + +public class ToStringTransformer implements Transformer { + + public static final ToStringTransformer instance = new ToStringTransformer(); + + @Override public Object transform(Object o) { return o == null ? "null" : o.toString(); } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/Topology.java b/src/main/java/org/cobbzilla/util/collection/Topology.java new file mode 100644 index 0000000..96f05d2 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/Topology.java @@ -0,0 +1,114 @@ +package org.cobbzilla.util.collection; + +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; + +import java.util.*; +import java.util.stream.Collectors; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; + +// adapted from: https://en.wikipedia.org/wiki/Topological_sorting +@NoArgsConstructor +public class Topology { + + private final List> nodes = new ArrayList<>(); + + public void addNode(T thing, Collection refs) { + final Node node = nodes.stream() + .filter(n -> n.thing.equals(thing)) + .findFirst() + .orElse(new Node<>(thing)); + + // add refs as edges; skip self-references + refs.stream().filter(ref -> !ref.equals(thing)).forEach(ref -> { + final Node existingEdgeNode = nodes.stream() + .filter(n -> n.thing.equals(ref)) + .findFirst() + .orElse(null); + if (existingEdgeNode != null) { + node.addEdge(existingEdgeNode); + } else { + final Node newEdgeNode = new Node<>(ref); + nodes.add(newEdgeNode); + node.addEdge(newEdgeNode); + } + }); + nodes.add(node); + } + + static class Node { + public final T thing; + public final HashSet> inEdges; + public final HashSet> outEdges; + + public Node(T thing) { + this.thing = thing; + inEdges = new HashSet<>(); + outEdges = new HashSet<>(); + } + public Node addEdge(Node node){ + final Edge e = new Edge<>(this, node); + outEdges.add(e); + node.inEdges.add(e); + return this; + } + public String toString() { return thing.toString(); } + } + + @AllArgsConstructor + static class Edge { + public final Node from; + public final Node to; + @Override public boolean equals(Object obj) { + final Edge e = (Edge) obj; + return e.from == from && e.to == to; + } + } + + public List sort() { + + // L <- Empty list that will contain the sorted elements + final ArrayList> L = new ArrayList<>(); + + // S <- Set of all nodes with no incoming edges + final HashSet> S = new HashSet<>(); + nodes.stream().filter(n -> n.inEdges.isEmpty()).forEach(S::add); + + // while S is non-empty do + while (!S.isEmpty()) { + // remove a node n from S + final Node n = S.iterator().next(); + S.remove(n); + + // insert n into L + L.add(n); + + // for each node m with an edge e from n to m do + for (Iterator> it = n.outEdges.iterator(); it.hasNext();) { + // remove edge e from the graph + final Edge e = it.next(); + final Node m = e.to; + it.remove();//Remove edge from n + m.inEdges.remove(e);//Remove edge from m + + // if m has no other incoming edges then insert m into S + if (m.inEdges.isEmpty()) S.add(m); + } + } + // Check to see if all edges are removed + for (Node n : nodes) { + if (!n.inEdges.isEmpty()) { + return die("Cycle present, topological sort not possible"); + } + } + return L.stream().map(n -> n.thing).collect(Collectors.toList()); + } + + public List sortReversed() { + final List sorted = sort(); + Collections.reverse(sorted); + return sorted; + } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/mappy/Mappy.java b/src/main/java/org/cobbzilla/util/collection/mappy/Mappy.java new file mode 100644 index 0000000..03080a1 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/mappy/Mappy.java @@ -0,0 +1,246 @@ +package org.cobbzilla.util.collection.mappy; + +import lombok.Getter; +import lombok.experimental.Accessors; +import org.cobbzilla.util.string.StringUtil; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +import static org.cobbzilla.util.reflect.ReflectionUtil.getTypeParam; + +/** + * Mappy is a map of keys to collections of values. The collection type is configurable and there are several + * subclasses available. See MappyList, MappySet, MappySortedSet, and MappyConcurrentSortedSet + * + * It can be viewed either as a mapping of K->V or as K->C->V + * + * Mappy objects are meant to be short-lived. While methods are generally thread-safe, the getter will create a new empty + * collection every time a key is not found. So it makes a horrible cache. Mappy instances are best suited to be value + * objects of limited scope. + * + * @param key class + * @param value class + * @param collection class + */ +@Accessors(chain=true) +public abstract class Mappy> implements Map { + + private final ConcurrentHashMap map; + + @Getter(lazy=true) private final Class valueClass = initValueClass(); + private Class initValueClass() { return getTypeParam(getClass(), 2); } + + public Mappy () { map = new ConcurrentHashMap<>(); } + public Mappy (int size) { map = new ConcurrentHashMap<>(size); } + + public Mappy(Map> other) { + this(); + for (Map.Entry> entry : other.entrySet()) { + putAll(entry.getKey(), entry.getValue()); + } + } + + /** + * For subclasses to override and provide their own collection types + * @return A new (empty) instance of the collection type + */ + protected abstract C newCollection(); + + /** + * @return the number of key mappings + */ + @Override public int size() { return map.size(); } + + /** + * @return the total number of values (may be higher than # of keys) + */ + public int totalSize () { + int count = 0; + for (Collection c : allValues()) count += c.size(); + return count; + } + + /** + * @return true if this Mappy contains no values. It may contain keys whose collections have no values. + */ + @Override public boolean isEmpty() { return flatten().isEmpty(); } + + @Override public boolean containsKey(Object key) { return map.containsKey(key); } + + /** + * @param value the value to check + * @return true if the Mappy contains any collection that contains the value, which should be of type V + */ + @Override public boolean containsValue(Object value) { + for (C collection : allValues()) { + //noinspection SuspiciousMethodCalls + if (collection.contains(value)) return true; + } + return false; + } + + /** + * @param key the key to find + * @return the first value in the collection for they key, or null if the collection is empty + */ + @Override public V get(Object key) { + final C collection = getAll((K) key); + return collection.isEmpty() ? null : firstInCollection(collection); + } + + protected V firstInCollection(C collection) { return collection.iterator().next(); } + + /** + * Get the collection of values for a key. This method never returns null. + * @param key the key to find + * @return the collection of values for the key, which may be empty + */ + public C getAll (K key) { + C collection = map.get(key); + if (collection == null) { + collection = newCollection(); + map.put(key, collection); + } + return collection; + } + + /** + * Add a mapping. + * @param key the key to add + * @param value the value to add + * @return the value passed in, if the map already contained the item. null otherwise. + */ + @Override public V put(K key, V value) { + V rval = null; + synchronized (map) { + C group = map.get(key); + if (group == null) { + group = newCollection(); + map.put(key, group); + } else { + rval = group.contains(value) ? value : null; + } + group.add(value); + } + return rval; + } + + /** + * Remove a key + * @param key the key to remove + * @return The first value in the collection that was referenced by the key + */ + @Override public V remove(Object key) { + final C group = map.remove(key); + if (group == null || group.isEmpty()) return null; // empty case should never happen, but just in case + return group.iterator().next(); + } + + /** + * Put a bunch of stuff into the map + * @param m mappings to add + */ + @Override public void putAll(Map m) { + for (Entry e : m.entrySet()) { + put(e.getKey(), e.getValue()); + } + } + + /** + * Put a bunch of stuff into the map + * @param key the key to add + * @param values the values to add to the key's collection + */ + public void putAll(K key, Collection values) { + synchronized (map) { + C collection = getAll(key); + if (collection == null) collection = newCollection(); + collection.addAll(values); + map.put(key, collection); + } + } + + /** + * Erase the entire map. + */ + @Override public void clear() { map.clear(); } + + @Override public Set keySet() { return map.keySet(); } + + @Override public Collection values() { + final List vals = new ArrayList<>(); + for (C collection : map.values()) vals.addAll(collection); + return vals; + } + @Override public Set> entrySet() { + final Set> entries = new HashSet<>(); + for (Entry entry : map.entrySet()) { + for (V item : entry.getValue()) { + entries.add(new AbstractMap.SimpleEntry(entry.getKey(), item)); + } + } + return entries; + } + + public Collection allValues() { return map.values(); } + public Set> allEntrySets() { return map.entrySet(); } + + public List flatten() { + final List values = new ArrayList<>(); + for (C collection : allValues()) values.addAll(collection); + return values; + } + + public List flatten(Collection values) { + for (C collection : allValues()) values.addAll(collection); + return new ArrayList<>(values); + } + + @Override public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + final Mappy other = (Mappy) o; + + if (totalSize() != other.totalSize()) return false; + + for (K key : keySet()) { + if (!other.containsKey(key)) return false; + final Collection otherValues = other.getAll(key); + final Collection thisValues = getAll(key); + if (otherValues.size() != thisValues.size()) return false; + for (Object value : thisValues) { + if (!otherValues.contains(value)) return false; + } + } + return true; + } + + @Override public int hashCode() { + int result = Integer.valueOf(totalSize()).hashCode(); + result = 31 * result + (valueClass != null ? valueClass.hashCode() : 0); + for (K key : keySet()) { + result = 31 * result + (key.hashCode() + 13); + for (V value : getAll(key)) { + result = 31 * result + (value == null ? 0 : value.hashCode()); + } + } + return result; + } + + @Override public String toString() { + final StringBuilder b = new StringBuilder(); + for (K key : keySet()) { + if (b.length() > 0) b.append(" | "); + b.append(key).append("->(").append(StringUtil.toString(getAll(key), ", ")).append(")"); + } + return "{"+b.toString()+"}"; + } + + public Map toMap() { + final HashMap m = new HashMap<>(); + for (K key : keySet()) m.put(key, getAll(key)); + return m; + } +} diff --git a/src/main/java/org/cobbzilla/util/collection/mappy/MappyConcurrentSortedSet.java b/src/main/java/org/cobbzilla/util/collection/mappy/MappyConcurrentSortedSet.java new file mode 100644 index 0000000..c5a9414 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/mappy/MappyConcurrentSortedSet.java @@ -0,0 +1,24 @@ +package org.cobbzilla.util.collection.mappy; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.util.Comparator; +import java.util.concurrent.ConcurrentSkipListSet; + +@NoArgsConstructor @AllArgsConstructor +public class MappyConcurrentSortedSet extends Mappy> { + + public MappyConcurrentSortedSet(int size) { super(size); } + + @Getter @Setter private Comparator comparator; + + @Override protected ConcurrentSkipListSet newCollection() { + return comparator == null ? new ConcurrentSkipListSet() : new ConcurrentSkipListSet<>(comparator); + } + + @Override protected V firstInCollection(ConcurrentSkipListSet collection) { return collection.first(); } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/mappy/MappyList.java b/src/main/java/org/cobbzilla/util/collection/mappy/MappyList.java new file mode 100644 index 0000000..8b582c2 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/mappy/MappyList.java @@ -0,0 +1,29 @@ +package org.cobbzilla.util.collection.mappy; + +import lombok.NoArgsConstructor; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +@NoArgsConstructor +public class MappyList extends Mappy> { + + protected Integer subSize; + + public MappyList (int size) { super(size); } + public MappyList (int size, int subSize) { super(size); this.subSize = subSize; } + + public MappyList(Map> other, Integer subSize) { + super(other); + this.subSize = subSize; + } + + public MappyList(Map> other) { this(other, null); } + + @Override protected List newCollection() { return subSize != null ? new ArrayList(subSize) : new ArrayList(); } + + @Override protected V firstInCollection(List collection) { return collection.get(0); } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/mappy/MappySet.java b/src/main/java/org/cobbzilla/util/collection/mappy/MappySet.java new file mode 100644 index 0000000..c39f673 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/mappy/MappySet.java @@ -0,0 +1,16 @@ +package org.cobbzilla.util.collection.mappy; + +import lombok.NoArgsConstructor; + +import java.util.HashSet; +import java.util.Set; + +@NoArgsConstructor +public class MappySet extends Mappy> { + + public MappySet (int size) { super(size); } + + @Override protected Set newCollection() { return new HashSet<>(); } + +} + diff --git a/src/main/java/org/cobbzilla/util/collection/mappy/MappySortedSet.java b/src/main/java/org/cobbzilla/util/collection/mappy/MappySortedSet.java new file mode 100644 index 0000000..080dd94 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/mappy/MappySortedSet.java @@ -0,0 +1,22 @@ +package org.cobbzilla.util.collection.mappy; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.util.Comparator; +import java.util.TreeSet; + +@NoArgsConstructor @AllArgsConstructor +public class MappySortedSet extends Mappy> { + + public MappySortedSet(int size) { super(size); } + + @Getter @Setter private Comparator comparator; + + @Override protected TreeSet newCollection() { return comparator == null ? new TreeSet() : new TreeSet<>(comparator); } + + @Override protected V firstInCollection(TreeSet collection) { return collection.first(); } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/mappy/MappyStringKeyList.java b/src/main/java/org/cobbzilla/util/collection/mappy/MappyStringKeyList.java new file mode 100644 index 0000000..57f58c7 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/mappy/MappyStringKeyList.java @@ -0,0 +1,39 @@ +package org.cobbzilla.util.collection.mappy; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.NoArgsConstructor; + +import java.util.Collection; +import java.util.Iterator; +import java.util.Map; + +import static java.util.Arrays.asList; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.reflect.ReflectionUtil.arrayClass; +import static org.cobbzilla.util.reflect.ReflectionUtil.getFirstTypeParam; + +@NoArgsConstructor +public class MappyStringKeyList extends MappyList { + + public MappyStringKeyList(int size) { super(size); } + + public MappyStringKeyList(int size, int subSize) { super(size, subSize); } + + public MappyStringKeyList(Map> other, Integer subSize) { + super(other); + this.subSize = subSize; + } + + public MappyStringKeyList(Map other) { this(other, null); } + + public MappyStringKeyList(String json) { + final ObjectNode object = json(json, ObjectNode.class); + final Class arrayClass = arrayClass(getFirstTypeParam(getClass())); + for (Iterator> iter = object.fields(); iter.hasNext(); ) { + final Map.Entry entry = iter.next(); + putAll(entry.getKey(), asList((V[]) json(entry.getValue(), arrayClass))); + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/multi/MultiResult.java b/src/main/java/org/cobbzilla/util/collection/multi/MultiResult.java new file mode 100644 index 0000000..ffceb99 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/multi/MultiResult.java @@ -0,0 +1,46 @@ +package org.cobbzilla.util.collection.multi; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class MultiResult { + + public List successes = new ArrayList<>(); + public Map failures = new LinkedHashMap<>(); + + public int successCount() { return successes.size(); } + public int failCount() { return failures.size(); } + + public void success(String name) { successes.add(name); } + public void fail(String name, String reason) { failures.put(name, reason); } + + public boolean hasFailures () { return !failures.isEmpty(); } + + public String getHeader() { return "TEST RESULTS"; } + + public String toString() { + final StringBuilder b = new StringBuilder(); + b.append("\n\n").append(getHeader()).append("\n--------------------\n") + .append(successCount()).append("\tsucceeded\n") + .append(failCount()).append("\tfailed"); + if (!failures.isEmpty()) { + b.append(":\n"); + for (String fail : failures.keySet()) { + b.append(fail).append("\n"); + } + b.append("--------------------\n"); + b.append("\nfailure details:\n"); + for (Map.Entry fail : failures.entrySet()) { + b.append(fail.getKey()).append(":\t").append(fail.getValue()).append("\n"); + b.append("--------\n"); + } + } else { + b.append("\n"); + } + b.append("--------------------\n"); + return b.toString(); + } + +} diff --git a/src/main/java/org/cobbzilla/util/collection/multi/MultiResultDriver.java b/src/main/java/org/cobbzilla/util/collection/multi/MultiResultDriver.java new file mode 100644 index 0000000..41a8aa0 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/multi/MultiResultDriver.java @@ -0,0 +1,30 @@ +package org.cobbzilla.util.collection.multi; + +import java.util.Map; + +public interface MultiResultDriver { + + MultiResult getResult (); + + // called before trying calculate result + void before (); + + void exec (Object task); + + // called if calculation was a success + void success (String message); + + // called if calculation failed + void failure (String message, Exception e); + + // called at the end (should via finally block) + void after (); + + // allows the caller/user to stash things for use during execution + Map getContext(); + void setContext(Map context); + + int getMaxConcurrent(); + long getTimeout(); + +} diff --git a/src/main/java/org/cobbzilla/util/collection/multi/MultiResultDriverBase.java b/src/main/java/org/cobbzilla/util/collection/multi/MultiResultDriverBase.java new file mode 100644 index 0000000..201a3c2 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/collection/multi/MultiResultDriverBase.java @@ -0,0 +1,32 @@ +package org.cobbzilla.util.collection.multi; + +import lombok.Getter; +import org.apache.commons.lang3.exception.ExceptionUtils; + +public abstract class MultiResultDriverBase implements MultiResultDriver { + + @Getter protected MultiResult result = new MultiResult(); + + protected abstract String successMessage(Object task); + protected abstract String failureMessage(Object task); + protected abstract void run(Object task) throws Exception; + + @Override public void before() {} + @Override public void after() {} + + @Override public void exec(Object task) { + try { + before(); + run(task); + success(successMessage(task)); + } catch (Exception e) { + failure(failureMessage(task), e); + } finally { + after(); + } + } + + @Override public void success(String message) { result.success(message); } + @Override public void failure(String message, Exception e) { result.fail(message, e.toString() + "\n- stack -\n" + ExceptionUtils.getStackTrace(e) + "\n- end stack -\n"); } + +} diff --git a/src/main/java/org/cobbzilla/util/cron/CronCommand.java b/src/main/java/org/cobbzilla/util/cron/CronCommand.java new file mode 100644 index 0000000..b37dc50 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/cron/CronCommand.java @@ -0,0 +1,12 @@ +package org.cobbzilla.util.cron; + +import java.util.Map; +import java.util.Properties; + +public interface CronCommand { + + void init (Properties properties); + + void exec (Map context) throws Exception; + +} diff --git a/src/main/java/org/cobbzilla/util/cron/CronCommandBase.java b/src/main/java/org/cobbzilla/util/cron/CronCommandBase.java new file mode 100644 index 0000000..d7023f3 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/cron/CronCommandBase.java @@ -0,0 +1,12 @@ +package org.cobbzilla.util.cron; + +import java.util.Properties; + +public abstract class CronCommandBase implements CronCommand { + + protected Properties properties; + + @Override + public void init(Properties properties) { this.properties = properties; } + +} diff --git a/src/main/java/org/cobbzilla/util/cron/CronDaemon.java b/src/main/java/org/cobbzilla/util/cron/CronDaemon.java new file mode 100644 index 0000000..edf1da1 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/cron/CronDaemon.java @@ -0,0 +1,13 @@ +package org.cobbzilla.util.cron; + +public interface CronDaemon { + + void start () throws Exception; + + void stop() throws Exception; + + void addJob(final CronJob job) throws Exception; + + void removeJob(final String id) throws Exception; + +} \ No newline at end of file diff --git a/src/main/java/org/cobbzilla/util/cron/CronJob.java b/src/main/java/org/cobbzilla/util/cron/CronJob.java new file mode 100644 index 0000000..5b7109b --- /dev/null +++ b/src/main/java/org/cobbzilla/util/cron/CronJob.java @@ -0,0 +1,29 @@ +package org.cobbzilla.util.cron; + +import lombok.Getter; +import lombok.Setter; +import org.cobbzilla.util.reflect.ReflectionUtil; + +import java.util.Properties; + +public class CronJob { + + @Getter @Setter private String id; + + @Getter @Setter private String cronTimeString; + + @Getter @Setter private boolean startNow = false; + + @Getter @Setter private String commandClass; + @Getter @Setter private Properties properties = new Properties(); + + // todo +// @Getter @Setter private String user; +// @Getter @Setter private String shellCommand; + + public CronCommand getCommandInstance() { + CronCommand command = ReflectionUtil.instantiate(commandClass); + command.init(properties); + return command; + } +} diff --git a/src/main/java/org/cobbzilla/util/cron/quartz/QuartzMaster.java b/src/main/java/org/cobbzilla/util/cron/quartz/QuartzMaster.java new file mode 100644 index 0000000..993e355 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/cron/quartz/QuartzMaster.java @@ -0,0 +1,74 @@ +package org.cobbzilla.util.cron.quartz; + +import lombok.Getter; +import lombok.Setter; +import org.cobbzilla.util.cron.CronCommand; +import org.cobbzilla.util.cron.CronDaemon; +import org.cobbzilla.util.cron.CronJob; +import org.quartz.*; +import org.quartz.impl.StdSchedulerFactory; + +import java.util.List; +import java.util.Map; +import java.util.TimeZone; + +import static org.quartz.CronScheduleBuilder.cronSchedule; +import static org.quartz.JobBuilder.newJob; +import static org.quartz.TriggerBuilder.newTrigger; + +public class QuartzMaster implements CronDaemon { + + private static final String CAL_SUFFIX = "_calendar"; + private static final String JOB_SUFFIX = "_jobDetail"; + private static final String TRIGGER_SUFFIX = "_trigger"; + + private Scheduler scheduler; + @Getter @Setter private TimeZone timeZone; + + @Getter @Setter private List jobs; + + public void start () throws Exception { + scheduler = StdSchedulerFactory.getDefaultScheduler(); + scheduler.start(); + + if (jobs != null) { + for (final CronJob job : jobs) { + addJob(job); + } + } + } + + public void addJob(final CronJob job) throws SchedulerException { + String id = job.getId(); + + Job specialJob = new Job () { + @Override + public void execute(JobExecutionContext context) throws JobExecutionException { + Map map = context.getMergedJobDataMap(); + try { + CronCommand command = job.getCommandInstance(); + command.init(job.getProperties()); + command.exec(map); + } catch (Exception e) { + throw new JobExecutionException(e); + } + } + }; + + final JobDetail jobDetail = newJob(specialJob.getClass()).withIdentity(id+JOB_SUFFIX).build(); + + TriggerBuilder builder = newTrigger().withIdentity(id+TRIGGER_SUFFIX); + if (job.isStartNow()) builder = builder.startNow(); + final CronScheduleBuilder cronSchedule = cronSchedule(job.getCronTimeString()); + final Trigger trigger = builder.withSchedule(timeZone != null ? cronSchedule.inTimeZone(timeZone) : cronSchedule).build(); + + scheduler.scheduleJob(jobDetail, trigger); + } + + @Override public void removeJob(final String id) throws Exception { + scheduler.deleteJob(new JobKey(id+JOB_SUFFIX)); + } + + public void stop () throws Exception { scheduler.shutdown(); } + +} diff --git a/src/main/java/org/cobbzilla/util/daemon/Await.java b/src/main/java/org/cobbzilla/util/daemon/Await.java new file mode 100644 index 0000000..1468dfa --- /dev/null +++ b/src/main/java/org/cobbzilla/util/daemon/Await.java @@ -0,0 +1,170 @@ +package org.cobbzilla.util.daemon; + +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.time.ClockProvider; + +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.now; +import static org.cobbzilla.util.system.Sleep.sleep; + +@Slf4j +public class Await { + + public static final long DEFAULT_AWAIT_GET_SLEEP = 10; + public static final long DEFAULT_AWAIT_RETRY_SLEEP = 100; + + public static E awaitFirst(Collection> futures, long timeout) throws TimeoutException { + return awaitFirst(futures, timeout, DEFAULT_AWAIT_RETRY_SLEEP); + } + public static E awaitFirst(Collection> futures, long timeout, long retrySleep) throws TimeoutException { + return awaitFirst(futures, timeout, retrySleep, DEFAULT_AWAIT_GET_SLEEP); + } + + public static E awaitFirst(Collection> futures, long timeout, long retrySleep, long getSleep) throws TimeoutException { + long start = now(); + while (!futures.isEmpty() && now() - start < timeout) { + for (Iterator> iter = futures.iterator(); iter.hasNext(); ) { + Future future = iter.next(); + try { + final E value = future.get(getSleep, TimeUnit.MILLISECONDS); + if (value != null) return value; + iter.remove(); + if (futures.isEmpty()) break; + + } catch (InterruptedException e) { + die("await: interrupted: " + e); + } catch (ExecutionException e) { + die("await: execution error: " + e); + } catch (TimeoutException e) { + // noop + } + sleep(retrySleep); + } + } + if (now() - start > timeout) throw new TimeoutException("await: timed out"); + return null; // all futures had a null result + } + + public static List awaitAndCollect(Collection> futures, int maxResults, long timeout) throws TimeoutException { + return awaitAndCollect(futures, maxResults, timeout, DEFAULT_AWAIT_RETRY_SLEEP); + } + + public static List awaitAndCollect(Collection> futures, int maxResults, long timeout, long retrySleep) throws TimeoutException { + return awaitAndCollect(futures, maxResults, timeout, retrySleep, DEFAULT_AWAIT_GET_SLEEP); + } + + public static List awaitAndCollect(Collection> futures, int maxResults, long timeout, long retrySleep, long getSleep) throws TimeoutException { + return awaitAndCollect(futures, maxResults, timeout, retrySleep, getSleep, new ArrayList()); + } + + public static List awaitAndCollect(List> futures, int maxQueryResults, long timeout, List results) throws TimeoutException { + return awaitAndCollect(futures, maxQueryResults, timeout, DEFAULT_AWAIT_RETRY_SLEEP, DEFAULT_AWAIT_GET_SLEEP, results); + } + + public static List awaitAndCollect(Collection> futures, int maxResults, long timeout, long retrySleep, long getSleep, List results) throws TimeoutException { + long start = now(); + int size = futures.size(); + while (!futures.isEmpty() && now() - start < timeout) { + for (Iterator> iter = futures.iterator(); iter.hasNext(); ) { + Future future = iter.next(); + try { + results.addAll((List) future.get(getSleep, TimeUnit.MILLISECONDS)); + iter.remove(); + if (--size <= 0 || results.size() >= maxResults) return results; + break; + + } catch (InterruptedException e) { + die("await: interrupted: " + e); + } catch (ExecutionException e) { + die("await: execution error: " + e); + } catch (TimeoutException e) { + // noop + } + sleep(retrySleep); + } + } + if (now() - start > timeout) throw new TimeoutException("await: timed out"); + return results; + } + + public static Set awaitAndCollectSet(Collection> futures, int maxResults, long timeout) throws TimeoutException { + return awaitAndCollectSet(futures, maxResults, timeout, DEFAULT_AWAIT_RETRY_SLEEP); + } + + public static Set awaitAndCollectSet(Collection> futures, int maxResults, long timeout, long retrySleep) throws TimeoutException { + return awaitAndCollectSet(futures, maxResults, timeout, retrySleep, DEFAULT_AWAIT_GET_SLEEP); + } + + public static Set awaitAndCollectSet(Collection> futures, int maxResults, long timeout, long retrySleep, long getSleep) throws TimeoutException { + return awaitAndCollectSet(futures, maxResults, timeout, retrySleep, getSleep, new HashSet()); + } + + public static Set awaitAndCollectSet(List> futures, int maxQueryResults, long timeout, Set results) throws TimeoutException { + return awaitAndCollectSet(futures, maxQueryResults, timeout, DEFAULT_AWAIT_RETRY_SLEEP, DEFAULT_AWAIT_GET_SLEEP, results); + } + + public static Set awaitAndCollectSet(Collection> futures, int maxResults, long timeout, long retrySleep, long getSleep, Set results) throws TimeoutException { + long start = now(); + int size = futures.size(); + while (!futures.isEmpty() && now() - start < timeout) { + for (Iterator> iter = futures.iterator(); iter.hasNext(); ) { + Future future = iter.next(); + try { + results.addAll((Collection) future.get(getSleep, TimeUnit.MILLISECONDS)); + iter.remove(); + if (--size <= 0 || results.size() >= maxResults) return results; + break; + + } catch (InterruptedException e) { + die("await: interrupted: " + e); + } catch (ExecutionException e) { + die("await: execution error: " + e); + } catch (TimeoutException e) { + // noop + } + sleep(retrySleep); + } + } + if (now() - start > timeout) throw new TimeoutException("await: timed out"); + return results; + } + + public static AwaitResult awaitAll(Collection> futures, long timeout) { + return awaitAll(futures, timeout, ClockProvider.SYSTEM); + } + + public static AwaitResult awaitAll(Collection> futures, long timeout, ClockProvider clock) { + long start = clock.now(); + final AwaitResult result = new AwaitResult<>(); + final Collection> awaiting = new ArrayList<>(futures); + + while (clock.now() - start < timeout) { + for (Iterator iter = awaiting.iterator(); iter.hasNext(); ) { + final Future f = (Future) iter.next(); + if (f.isDone()) { + iter.remove(); + try { + final T r = (T) f.get(); + if (r != null) log.info("awaitAll: "+ r); + result.success(f, r); + + } catch (Exception e) { + log.warn("awaitAll: "+e, e); + result.fail(f, e); + } + } + } + if (awaiting.isEmpty()) break; + sleep(200); + } + + result.timeout(awaiting); + return result; + } +} diff --git a/src/main/java/org/cobbzilla/util/daemon/AwaitResult.java b/src/main/java/org/cobbzilla/util/daemon/AwaitResult.java new file mode 100644 index 0000000..3431cd8 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/daemon/AwaitResult.java @@ -0,0 +1,38 @@ +package org.cobbzilla.util.daemon; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; + +import java.util.*; +import java.util.concurrent.Future; + +public class AwaitResult { + + @Getter private Map successes = new HashMap<>(); + public void success(Future f, T thing) { successes.put(f, thing); } + public int numSuccesses () { return successes.size(); } + + @Getter private Map failures = new HashMap<>(); + public void fail(Future f, Exception e) { failures.put(f, e); } + public int numFails () { return failures.size(); } + + @Getter private List timeouts = new ArrayList<>(); + public void timeout (Collection> timedOut) { timeouts.addAll(timedOut); } + public boolean timedOut() { return !timeouts.isEmpty(); } + public int numTimeouts () { return timeouts.size(); } + + public boolean allSucceeded() { return failures.isEmpty() && timeouts.isEmpty(); } + + @JsonIgnore public List getNotNullSuccesses() { + final List ok = new ArrayList<>(); + for (T t : getSuccesses().values()) if (t != null) ok.add(t); + return ok; + } + + public String toString() { + return "successes=" + successes.size() + + ", failures=" + failures.size() + + ", timeouts=" + timeouts.size(); + } + +} diff --git a/src/main/java/org/cobbzilla/util/daemon/BufferedRunDaemon.java b/src/main/java/org/cobbzilla/util/daemon/BufferedRunDaemon.java new file mode 100644 index 0000000..be9ff3e --- /dev/null +++ b/src/main/java/org/cobbzilla/util/daemon/BufferedRunDaemon.java @@ -0,0 +1,83 @@ +package org.cobbzilla.util.daemon; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import static java.util.concurrent.TimeUnit.HOURS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.cobbzilla.util.daemon.ZillaRuntime.background; +import static org.cobbzilla.util.daemon.ZillaRuntime.now; +import static org.cobbzilla.util.system.Sleep.nap; +import static org.cobbzilla.util.time.TimeUtil.formatDuration; + +@AllArgsConstructor @Slf4j +public class BufferedRunDaemon implements Runnable { + + public static final long IDLE_SYNC_INTERVAL = HOURS.toMillis(1); + public static final long MIN_SYNC_WAIT = SECONDS.toMillis(10); + + private final String logPrefix; + private final Runnable action; + + private final AtomicReference daemonThread = new AtomicReference<>(); + private final AtomicLong lastRun = new AtomicLong(0); + private final AtomicLong lastRunRequested = new AtomicLong(0); + + private final AtomicBoolean done = new AtomicBoolean(false); + + protected long getIdleSyncInterval() { return IDLE_SYNC_INTERVAL; } + protected long getMinSyncWait () { return MIN_SYNC_WAIT; } + + public void start () { daemonThread.set(background(this)); } + + protected void interrupt() { if (daemonThread.get() != null) daemonThread.get().interrupt(); } + + public void poke () { lastRunRequested.set(now()); interrupt(); } + public void done () { done.set(true); interrupt(); } + + @Override public void run () { + long napTime; + + //noinspection InfiniteLoopStatement + while (true) { + napTime = getIdleSyncInterval(); + log.info(logPrefix+": sleep for "+formatDuration(napTime)+" awaiting activity"); + if (!nap(napTime, logPrefix+" napping for "+formatDuration(napTime)+" awaiting activity")) { + log.info(logPrefix + " interrupted during initial pause, continuing"); + } else { + boolean shouldDoIdleSleep = lastRunRequested.get() == 0; + if (shouldDoIdleSleep) { + shouldDoIdleSleep = lastRunRequested.get() == 0; + while (shouldDoIdleSleep && lastRun.get() > 0 && now() - lastRun.get() < getIdleSyncInterval()) { + log.info(logPrefix + " napping for " + formatDuration(napTime) + " due to no activity"); + if (!nap(napTime, logPrefix + " idle loop sleep")) { + log.info(logPrefix + " nap was interrupted, breaking out"); + break; + } + shouldDoIdleSleep = lastRunRequested.get() == 0; + } + } + } + + final long minSyncWait = getMinSyncWait(); + while (lastRunRequested.get() > 0 && now() - lastRunRequested.get() < minSyncWait) { + napTime = minSyncWait / 4; + log.info(logPrefix+" napping for "+formatDuration(napTime)+", waiting for at least "+formatDuration(minSyncWait)+" of no activity before starting sync"); + nap(napTime, logPrefix + " waiting for inactivity"); + } + + try { + action.run(); + } catch (Exception e) { + log.error(logPrefix+" sync: " + e, e); + } finally { + lastRun.set(now()); + lastRunRequested.set(0); + } + } + } +} diff --git a/src/main/java/org/cobbzilla/util/daemon/DaemonThreadFactory.java b/src/main/java/org/cobbzilla/util/daemon/DaemonThreadFactory.java new file mode 100644 index 0000000..e8aaf0a --- /dev/null +++ b/src/main/java/org/cobbzilla/util/daemon/DaemonThreadFactory.java @@ -0,0 +1,28 @@ +package org.cobbzilla.util.daemon; + +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; + +@Slf4j +public class DaemonThreadFactory implements ThreadFactory { + + public static final DaemonThreadFactory instance = new DaemonThreadFactory(); + + @Override public Thread newThread(Runnable r) { + final Thread t = new Thread(r); + t.setDaemon(true); + return t; + } + + public static ExecutorService fixedPool (int count) { + if (count <= 0) { + log.warn("fixedPool: invalid count ("+count+"), using single thread"); + count = 1; + } + return Executors.newFixedThreadPool(count, instance); + } + +} diff --git a/src/main/java/org/cobbzilla/util/daemon/ErrorApi.java b/src/main/java/org/cobbzilla/util/daemon/ErrorApi.java new file mode 100644 index 0000000..c5f87ac --- /dev/null +++ b/src/main/java/org/cobbzilla/util/daemon/ErrorApi.java @@ -0,0 +1,12 @@ +package org.cobbzilla.util.daemon; + +/** + * A generic interface for error reporting services like Errbit and Airbrake + */ +public interface ErrorApi { + + void report(Exception e); + void report(String s); + void report(String s, Exception e); + +} diff --git a/src/main/java/org/cobbzilla/util/daemon/SimpleDaemon.java b/src/main/java/org/cobbzilla/util/daemon/SimpleDaemon.java new file mode 100755 index 0000000..454fef3 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/daemon/SimpleDaemon.java @@ -0,0 +1,181 @@ +package org.cobbzilla.util.daemon; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import static org.cobbzilla.util.daemon.ZillaRuntime.now; +import static org.cobbzilla.util.system.Sleep.sleep; + +@Slf4j +public abstract class SimpleDaemon implements Runnable { + + public static final DateTimeFormatter DFORMAT = DateTimeFormat.forPattern("yyyy-MMM-dd HH:mm:ss"); + + public SimpleDaemon () { this.name = getClass().getSimpleName(); } + + public SimpleDaemon (String name) { this.name = name; } + + @Getter private String name; + @Getter private long lastProcessTime = 0; + + private volatile Thread mainThread = null; + private final Object lock = new Object(); + private volatile boolean isDone = false; + + /** Called right after daemon has started */ + public void onStart () {} + + /** Called right before daemon is about to exit */ + public void onStop () {} + + public void start() { + log.info(name+": Starting daemon"); + synchronized (lock) { + if (mainThread != null) { + log.warn(name+": daemon is already running, not starting it again"); + return; + } + mainThread = new Thread(this); + mainThread.setName(name); + } + mainThread.setDaemon(true); + mainThread.start(); + } + + private boolean alreadyStopped() { + if (mainThread == null) { + log.warn(name+": daemon is already stopped"); + return true; + } + return false; + } + + public void stop() { + if (alreadyStopped()) return; + isDone = true; + mainThread.interrupt(); + // Let's leave it at that, this thread is a daemon anyway. + } + + public void interrupt() { + if (alreadyStopped()) return; + mainThread.interrupt(); + } + + /** + * @deprecated USE WITH CAUTION -- calls Thread.stop() !! + */ + private void kill() { + if (alreadyStopped()) return; + isDone = true; + mainThread.stop(); + } + + /** + * Tries to stop the daemon. If it doesn't stop within "wait" millis, + * it gets killed. + */ + public void stopWithPossibleKill(long wait) { + stop(); + long start = now(); + while (getIsAlive() + && (now() - start < wait)) { + wait(25, "stopWithPossibleKill"); + } + if (getIsAlive()) { + kill(); + } + } + + protected void init() throws Exception {} + + public void run() { + onStart(); + long delay = getStartupDelay(); + if (delay > 0) { + log.debug(name + ": Delaying daemon startup for " + delay + "ms..."); + if (!wait(delay, "run[startup-delay]")) { + if (!canInterruptSleep()) return; + } + } + log.debug(name + ": Daemon thread now running"); + + try { + log.debug(name + ": Daemon thread invoking init"); + init(); + + while (!isDone) { + log.debug(name + ": Daemon thread invoking process"); + try { + process(); + lastProcessTime = now(); + } catch (Exception e) { + processException(e); + continue; + } + if (isDone) return; + if (!wait(getSleepTime(), "run[post-processing]")) { + if (canInterruptSleep()) continue; + return; + } + } + } catch (Exception e) { + log.error(name + ": Error in daemon, exiting: " + e, e); + + } finally { + cleanup(); + try { + onStop(); + } catch (Exception e) { + log.error(name + ": Error in onStop, exiting and ignoring error: " + e, e); + } + } + } + + public void processException(Exception e) throws Exception { throw e; } + + protected boolean wait(long delay, String reason) { + try { + sleep(delay, reason); + return true; + } catch (RuntimeException e) { + if (isDone) { + log.info("sleep("+delay+") interrupted but daemon is done"); + } else { + log.error("sleep("+delay+") interrupted, exiting: "+e); + } + return false; + } + } + + protected boolean canInterruptSleep() { return false; } + + protected long getStartupDelay() { return 0; } + + protected abstract long getSleepTime(); + + protected abstract void process(); + + public boolean getIsDone() { return isDone; } + + public boolean getIsAlive() { + try { + return mainThread != null && mainThread.isAlive(); + } catch (NullPointerException npe) { + return false; + } + } + + private void cleanup() { + mainThread = null; + isDone = true; + } + + public String getStatus() { + return "isDone=" + getIsDone() + + "\nlastProcessTime=" + DFORMAT.print(lastProcessTime) + + "\nsleepTime=" + getSleepTime()+"ms"; + } +} diff --git a/src/main/java/org/cobbzilla/util/daemon/ZillaRuntime.java b/src/main/java/org/cobbzilla/util/daemon/ZillaRuntime.java new file mode 100644 index 0000000..2b5374b --- /dev/null +++ b/src/main/java/org/cobbzilla/util/daemon/ZillaRuntime.java @@ -0,0 +1,407 @@ +package org.cobbzilla.util.daemon; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomUtils; +import org.apache.commons.lang3.SystemUtils; +import org.cobbzilla.util.collection.ToStringTransformer; +import org.cobbzilla.util.error.GeneralErrorHandler; +import org.cobbzilla.util.io.StreamUtil; +import org.cobbzilla.util.string.StringUtil; +import org.slf4j.Logger; + +import java.io.*; +import java.lang.management.ManagementFactory; +import java.lang.reflect.Array; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; + +import static java.lang.Long.toHexString; +import static java.util.concurrent.TimeUnit.SECONDS; +import static java.util.stream.LongStream.range; +import static org.apache.commons.collections.CollectionUtils.collect; +import static org.apache.commons.lang3.exception.ExceptionUtils.getStackTrace; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.io.FileUtil.list; +import static org.cobbzilla.util.reflect.ReflectionUtil.instantiate; +import static org.cobbzilla.util.string.StringUtil.truncate; +import static org.cobbzilla.util.system.Sleep.sleep; +import static org.cobbzilla.util.time.TimeUtil.formatDuration; + +/** + * the Zilla doesn't mess around. + */ +@Slf4j +public class ZillaRuntime { + + public static final String CLASSPATH_PREFIX = "classpath:"; + + public static String getJava() { return System.getProperty("java.home") + "/bin/java"; } + + public static boolean terminate(Thread thread, long timeout) { + if (thread == null || !thread.isAlive()) return true; + thread.interrupt(); + final long start = realNow(); + while (thread.isAlive() && realNow() - start < timeout) { + sleep(100, "terminate: waiting for thread to die: "+thread); + } + if (thread.isAlive()) { + log.warn("terminate: thread did not die voluntarily, killing it: "+thread); + thread.stop(); + } + return false; + } + + public static boolean bool(Boolean b) { return b != null && b; } + public static boolean bool(Boolean b, boolean val) { return b != null ? b : val; } + + public interface ExceptionRunnable { void handle(Exception e); } + + public static final ExceptionRunnable DEFAULT_EX_RUNNABLE = e -> log.error("Error: " + e); + + public static ExceptionRunnable exceptionRunnable (Class[] fatalExceptionClasses) { + return e -> { + for (Class c : fatalExceptionClasses) { + if (c.isAssignableFrom(e.getClass())) { + if (e instanceof RuntimeException) throw (RuntimeException) e; + die("fatal exception: "+e); + } + } + DEFAULT_EX_RUNNABLE.handle(e); + }; + } + + public static Thread background (Runnable r) { return background(r, DEFAULT_EX_RUNNABLE); } + + public static Thread background (Runnable r, ExceptionRunnable ex) { + final Thread t = new Thread(() -> { + try { + r.run(); + } catch (Exception e) { + ex.handle(e); + } + }); + t.start(); + return t; + } + + public static final Function DEFAULT_RETRY_BACKOFF = SECONDS::toMillis; + + public static T retry (Callable func, int tries) { + return retry(func, tries, DEFAULT_RETRY_BACKOFF, DEFAULT_EX_RUNNABLE); + } + + public static T retry (Callable func, int tries, Function backoff) { + return retry(func, tries, backoff, DEFAULT_EX_RUNNABLE); + } + + public static T retry (Callable func, + int tries, + Logger logger) { + return retry(func, tries, DEFAULT_RETRY_BACKOFF, e -> logger.error("Error: "+e)); + } + + public static T retry (Callable func, + int tries, + Function backoff, + Logger logger) { + return retry(func, tries, backoff, e -> logger.error("Error: "+e)); + } + + public static T retry (Callable func, + int tries, + Function backoff, + ExceptionRunnable ex) { + Exception lastEx = null; + try { + for (int i = 0; i < tries; i++) { + try { + final T rVal = func.call(); + log.debug("retry: successful, returning: " + rVal); + return rVal; + } catch (Exception e) { + lastEx = e; + log.debug("retry: failed (attempt " + (i + 1) + "/" + tries + "): " + e); + ex.handle(e); + sleep(backoff.apply(i), "waiting to retry " + func.getClass().getSimpleName()); + } + } + } catch (Exception e) { + return die("retry: fatal exception, exiting: "+e); + } + return die("retry: max tries ("+tries+") exceeded. last exception: "+lastEx); + } + + public static Thread daemon (Runnable r) { + final Thread t = new Thread(r); + t.setDaemon(true); + t.start(); + return t; + } + + @Getter @Setter private static ErrorApi errorApi; + + public static T die(String message) { return _throw(new IllegalStateException(message, null)); } + public static T die(String message, Exception e) { return _throw(new IllegalStateException(message, e)); } + public static T die(Exception e) { return _throw(new IllegalStateException("(no message)", e)); } + + public static T notSupported() { return notSupported("not supported"); } + public static T notSupported(String message) { return _throw(new UnsupportedOperationException(message)); } + + private static T _throw (RuntimeException e) { + final String message = e.getMessage(); + final Throwable cause = e.getCause(); + if (errorApi != null) { + if (cause != null && cause instanceof Exception) errorApi.report(message, (Exception) cause); + else errorApi.report(e); + } + if (cause != null) log.error("Inner exception: " + message, cause); + throw e; + } + + public static String errorString(Exception e) { return errorString(e, 1000); } + + public static String errorString(Exception e, int maxlen) { + return truncate(e.getClass().getName()+": "+e.getMessage()+"\n"+ getStackTrace(e), maxlen); + } + + public static boolean empty(String s) { return s == null || s.length() == 0; } + + /** + * Determines if the parameter is "empty", by criteria described in @return + * Tries to avoid throwing exceptions, handling just about any case in a true/false fashion. + * + * @param o anything + * @return true if and only o is: + * * null + * * a collection, map, iterable or array that contains no objects + * * a file that does not exist or whose size is zero + * * a directory that does not exist or that contains no files + * * any object whose .toString method returns a zero-length string + */ + public static boolean empty(Object o) { + if (o == null) return true; + if (o instanceof String) return o.toString().length() == 0; + if (o instanceof Collection) return ((Collection)o).isEmpty(); + if (o instanceof Map) return ((Map)o).isEmpty(); + if (o instanceof JsonNode) { + if (o instanceof ObjectNode) return ((ObjectNode) o).size() == 0; + if (o instanceof ArrayNode) return ((ArrayNode) o).size() == 0; + final String json = ((JsonNode) o).textValue(); + return json == null || json.length() == 0; + } + if (o instanceof Iterable) return !((Iterable)o).iterator().hasNext(); + if (o instanceof File) { + final File f = (File) o; + return !f.exists() || f.length() == 0 || (f.isDirectory() && list(f).length == 0); + } + if (o.getClass().isArray()) { + if (o.getClass().getComponentType().isPrimitive()) { + switch (o.getClass().getComponentType().getName()) { + case "boolean": return ((boolean[]) o).length == 0; + case "byte": return ((byte[]) o).length == 0; + case "short": return ((short[]) o).length == 0; + case "char": return ((char[]) o).length == 0; + case "int": return ((int[]) o).length == 0; + case "long": return ((long[]) o).length == 0; + case "float": return ((float[]) o).length == 0; + case "double": return ((double[]) o).length == 0; + default: return o.toString().length() == 0; + } + } else { + return ((Object[]) o).length == 0; + } + } + return o.toString().length() == 0; + } + + public static T first (Iterable o) { return (T) ((Iterable) o).iterator().next(); } + public static T first (Map o) { return first(o.values()); } + public static T first (T[] o) { return o[0]; } + + public static T sorted(T o) { + if (empty(o)) return o; + if (o.getClass().isArray()) { + final Object[] copy = (Object[]) Array.newInstance(o.getClass().getComponentType(), + ((Object[])o).length); + System.arraycopy(o, 0, copy, 0 , copy.length); + Arrays.sort(copy); + return (T) copy; + } + if (o instanceof Collection) { + final List list = new ArrayList((Collection) o); + Collections.sort(list); + final Collection copy = (Collection) instantiate(o.getClass()); + copy.addAll(list); + return (T) copy; + } + return die("sorted: cannot sort a "+o.getClass().getSimpleName()+", can only sort arrays and Collections"); + } + public static List sortedList(T o) { + if (o == null) return null; + if (o instanceof Collection) return new ArrayList((Collection) o); + if (o instanceof Object[]) return Arrays.asList((Object[]) o); + return die("sortedList: cannot sort a "+o.getClass().getSimpleName()+", can only sort arrays and Collections"); + } + + public static Boolean safeBoolean(String val, Boolean ifNull) { return empty(val) ? ifNull : Boolean.valueOf(val); } + public static Boolean safeBoolean(String val) { return safeBoolean(val, null); } + + public static Integer safeInt(String val, Integer ifNull) { return empty(val) ? ifNull : Integer.valueOf(val); } + public static Integer safeInt(String val) { return safeInt(val, null); } + + public static Long safeLong(String val, Long ifNull) { return empty(val) ? ifNull : Long.valueOf(val); } + public static Long safeLong(String val) { return safeLong(val, null); } + + public static BigInteger bigint(long val) { return new BigInteger(String.valueOf(val)); } + public static BigInteger bigint(int val) { return new BigInteger(String.valueOf(val)); } + public static BigInteger bigint(byte val) { return new BigInteger(String.valueOf(val)); } + + public static BigDecimal big(String val) { return new BigDecimal(val); } + public static BigDecimal big(double val) { return new BigDecimal(String.valueOf(val)); } + public static BigDecimal big(float val) { return new BigDecimal(String.valueOf(val)); } + public static BigDecimal big(long val) { return new BigDecimal(String.valueOf(val)); } + public static BigDecimal big(int val) { return new BigDecimal(String.valueOf(val)); } + public static BigDecimal big(byte val) { return new BigDecimal(String.valueOf(val)); } + + public static int percent(int value, double pct) { return percent(value, pct, RoundingMode.HALF_UP); } + + public static int percent(int value, double pct, RoundingMode rounding) { + return big(value).multiply(big(pct)).setScale(0, rounding).intValue(); + } + + public static int percent(BigDecimal value, BigDecimal pct) { + return percent(value.intValue(), pct.multiply(big(0.01)).doubleValue(), RoundingMode.HALF_UP); + } + + public static String uuid() { return UUID.randomUUID().toString(); } + + @Getter @Setter private static volatile long systemTimeOffset = 0; + public static long now() { return System.currentTimeMillis() + systemTimeOffset; } + public static String hexnow() { return toHexString(now()); } + public static String hexnow(long now) { return toHexString(now); } + public static long realNow() { return System.currentTimeMillis(); } + + public static T pickRandom(T[] things) { return things[RandomUtils.nextInt(0, things.length)]; } + public static T pickRandom(List things) { return things.get(RandomUtils.nextInt(0, things.size())); } + + public static BufferedReader stdin() { return new BufferedReader(new InputStreamReader(System.in)); } + public static BufferedWriter stdout() { return new BufferedWriter(new OutputStreamWriter(System.out)); } + + public static String readStdin() { return StreamUtil.toStringOrDie(System.in); } + + public static int envInt (String name, int defaultValue) { return envInt(name, defaultValue, null, null); } + public static int envInt (String name, int defaultValue, Integer maxValue) { return envInt(name, defaultValue, null, maxValue); } + public static int envInt (String name, int defaultValue, Integer minValue, Integer maxValue) { + return envInt(name, defaultValue, minValue, maxValue, System.getenv()); + } + public static int envInt (String name, int defaultValue, Integer minValue, Integer maxValue, Map env) { + final String s = env.get(name); + if (!empty(s)) { + try { + final int val = Integer.parseInt(s); + if (val <= 0) { + log.warn("envInt: invalid value("+name+"): " +val+", returning "+defaultValue); + return defaultValue; + } else if (maxValue != null && val > maxValue) { + log.warn("envInt: value too large ("+name+"): " +val+ ", returning " + maxValue); + return maxValue; + } else if (minValue != null && val < minValue) { + log.warn("envInt: value too small ("+name+"): " +val+ ", returning " + minValue); + return minValue; + } + return val; + } catch (Exception e) { + log.warn("envInt: invalid value("+name+"): " +s+", returning "+defaultValue); + return defaultValue; + } + } + return defaultValue; + } + + public static int processorCount() { return Runtime.getRuntime().availableProcessors(); } + + public static String hashOf (Object... things) { + final StringBuilder b = new StringBuilder(); + for (Object thing : things) { + b.append(thing == null ? "null" : thing).append(":::"); + } + return b.toString(); + } + + public static Collection stringRange(Number start, Number end) { + return collect(range(start.longValue(), end.longValue()).boxed().iterator(), ToStringTransformer.instance); + } + + public static String zcat() { return SystemUtils.IS_OS_MAC ? "gzcat" : "zcat"; } + public static String zcat(File f) { return (SystemUtils.IS_OS_MAC ? "gzcat" : "zcat") + " " + abs(f); } + + public static final String[] OMIT_DEBUG_OPTIONS = {"-Xdebug", "-agentlib", "-Xrunjdwp"}; + + public static boolean isDebugOption (String arg) { + for (String opt : OMIT_DEBUG_OPTIONS) if (arg.startsWith(opt)) return true; + return false; + } + + public static String javaOptions() { return javaOptions(true); } + + public static String javaOptions(boolean excludeDebugOptions) { + final List opts = new ArrayList<>(); + for (String arg : ManagementFactory.getRuntimeMXBean().getInputArguments()) { + if (excludeDebugOptions && isDebugOption(arg)) continue; + opts.add(arg); + } + return StringUtil.toString(opts, " "); + } + + public static T dcl (AtomicReference target, Callable init) { + return dcl(target, init, null); + } + + @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") + public static T dcl (AtomicReference target, Callable init, GeneralErrorHandler error) { + if (target.get() == null) { + synchronized (target) { + if (target.get() == null) { + try { + target.set(init.call()); + } catch (Exception e) { + if (error != null) { + error.handleError("dcl: error initializing: "+e, e); + } else { + log.warn("dcl: "+e); + return null; + } + } + } + } + } + return target.get(); + } + + public static String stacktrace() { return getStackTrace(new Exception()); } + + private static final AtomicLong selfDestructInitiated = new AtomicLong(-1); + public static void setSelfDestruct (long t) { setSelfDestruct(t, 0); } + public static void setSelfDestruct (long t, int status) { + synchronized (selfDestructInitiated) { + final long dieTime = selfDestructInitiated.get(); + if (dieTime == -1) { + selfDestructInitiated.set(now()+t); + daemon(() -> { sleep(t); System.exit(status); }); + } else { + log.warn("setSelfDestruct: already set: self-destructing in "+formatDuration(dieTime-now())); + } + } + } +} diff --git a/src/main/java/org/cobbzilla/util/dns/DnsManager.java b/src/main/java/org/cobbzilla/util/dns/DnsManager.java new file mode 100644 index 0000000..edc73b1 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/dns/DnsManager.java @@ -0,0 +1,33 @@ +package org.cobbzilla.util.dns; + +import java.util.List; + +public interface DnsManager { + + /** + * List matching DNS records + * @param match The DnsRecordMatch query + * @return a List of DnsRecords that match + */ + List list(DnsRecordMatch match) throws Exception; + + /** + * Write a DNS record + * @param record a DNS record to create or update + * @return true if the record was written, false if it was not (it may have been unchanged) + */ + boolean write(DnsRecord record) throws Exception; + + /** + * Publish changes to DNS records. Must be called after calling write if you want to see the changes publicly. + */ + void publish() throws Exception; + + /** + * Delete matching DNS records + * @param match The DnsRecordMatch query + * @return A count of the number of records deleted, or -1 if this DnsManager does not support returning counts + */ + int remove(DnsRecordMatch match) throws Exception; + +} diff --git a/src/main/java/org/cobbzilla/util/dns/DnsRecord.java b/src/main/java/org/cobbzilla/util/dns/DnsRecord.java new file mode 100644 index 0000000..231fb58 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/dns/DnsRecord.java @@ -0,0 +1,131 @@ +package org.cobbzilla.util.dns; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.Accessors; +import org.cobbzilla.util.string.StringUtil; + +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static java.util.Comparator.comparing; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.dns.DnsType.A; +import static org.cobbzilla.util.dns.DnsType.SOA; + +@NoArgsConstructor @Accessors(chain=true) @ToString(callSuper=true) +public class DnsRecord extends DnsRecordBase { + + public static final int DEFAULT_TTL = (int) TimeUnit.HOURS.toSeconds(1); + + public static final String OPT_MX_RANK = "rank"; + public static final String OPT_NS_NAME = "ns"; + + public static final String OPT_SOA_MNAME = "mname"; + public static final String OPT_SOA_RNAME = "rname"; + public static final String OPT_SOA_SERIAL = "serial"; + public static final String OPT_SOA_REFRESH = "refresh"; + public static final String OPT_SOA_RETRY = "retry"; + public static final String OPT_SOA_EXPIRE = "expire"; + public static final String OPT_SOA_MINIMUM = "minimum"; + + public static final String[] MX_REQUIRED_OPTIONS = {OPT_MX_RANK}; + public static final String[] NS_REQUIRED_OPTIONS = {OPT_NS_NAME}; + public static final String[] SOA_REQUIRED_OPTIONS = { + OPT_SOA_MNAME, OPT_SOA_RNAME, OPT_SOA_SERIAL, OPT_SOA_REFRESH, OPT_SOA_EXPIRE, OPT_SOA_RETRY + }; + + public static final Comparator DUPE_COMPARATOR = comparing(DnsRecord::dnsUniq); + + @Getter @Setter private int ttl = DEFAULT_TTL; + @Getter @Setter private Map options; + public boolean hasOptions () { return options != null && !options.isEmpty(); } + + public DnsRecord (DnsType type, String fqdn, String value, int ttl) { + setType(type); + setFqdn(fqdn); + setValue(value); + setTtl(ttl); + } + + public static DnsRecord A(String host, String ip) { + return (DnsRecord) new DnsRecord().setType(A).setFqdn(host).setValue(ip); + } + + public DnsRecord setOption(String optName, String value) { + if (options == null) options = new HashMap<>(); + options.put(optName, value); + return this; + } + + public String getOption(String optName) { return options == null ? null : options.get(optName); } + + public int getIntOption(String optName, int defaultValue) { + try { + return Integer.parseInt(options.get(optName)); + } catch (Exception ignored) { + return defaultValue; + } + } + + @JsonIgnore public String[] getRequiredOptions () { + switch (getType()) { + case MX: return MX_REQUIRED_OPTIONS; + case NS: return NS_REQUIRED_OPTIONS; + case SOA: return SOA_REQUIRED_OPTIONS; + default: return StringUtil.EMPTY_ARRAY; + } + } + + @JsonIgnore public boolean hasAllRequiredOptions () { + for (String opt : getRequiredOptions()) { + if (options == null || !options.containsKey(opt)) return false; + } + return true; + } + + public String getOptions_string(String sep) { + final StringBuilder b = new StringBuilder(); + if (options != null) { + for (Map.Entry e : options.entrySet()) { + if (b.length() > 0) b.append(sep); + if (empty(e.getValue())) { + b.append(e.getKey()).append("=true"); + } else { + b.append(e.getKey()).append("=").append(e.getValue()); + } + } + } + return b.toString(); + } + + public DnsRecord setOptions_string(String arg) { + if (options == null) options = new HashMap<>(); + if (empty(arg)) return this; + + for (String kvPair : arg.split(",")) { + int eqPos = kvPair.indexOf("="); + if (eqPos == kvPair.length()) throw new IllegalArgumentException("Option cannot end in '=' character"); + if (eqPos == -1) { + options.put(kvPair.trim(), "true"); + } else { + options.put(kvPair.substring(0, eqPos).trim(), kvPair.substring(eqPos+1).trim()); + } + } + return this; + } + + public String dnsUniq() { return type == SOA ? SOA+":"+fqdn : dnsFormat(",", "|"); } + + public String dnsFormat() { + return dnsFormat(",", "|"); + } + public String dnsFormat(String fieldSep, String optionsSep) { + return getType().name().toUpperCase()+fieldSep+getFqdn()+fieldSep+getValue()+fieldSep+getTtl()+fieldSep+(!hasOptions() ? "" : getOptions_string(optionsSep)); + } +} diff --git a/src/main/java/org/cobbzilla/util/dns/DnsRecordBase.java b/src/main/java/org/cobbzilla/util/dns/DnsRecordBase.java new file mode 100644 index 0000000..cc3e97c --- /dev/null +++ b/src/main/java/org/cobbzilla/util/dns/DnsRecordBase.java @@ -0,0 +1,39 @@ +package org.cobbzilla.util.dns; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.*; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; + +import static org.apache.commons.lang3.StringUtils.chop; +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +@NoArgsConstructor @AllArgsConstructor @Accessors(chain=true) +@ToString @EqualsAndHashCode @Slf4j +public class DnsRecordBase { + + @Getter @Setter protected String fqdn; + public boolean hasFqdn() { return !empty(fqdn); } + + @JsonIgnore public String getNormalFqdn() { return empty(fqdn) ? fqdn : fqdn.endsWith(".") ? chop(fqdn) : fqdn; } + + public String getHost (String suffix) { + if (!hasFqdn()) return die("getHost: fqdn not set"); + if (getFqdn().endsWith(suffix)) return getFqdn().substring(0, getFqdn().length() - suffix.length() - 1); + log.warn("getHost: suffix mismatch: fqdn "+getFqdn()+" does not end with "+suffix); + return getFqdn(); + } + + @Getter @Setter protected DnsType type; + public boolean hasType () { return type != null; } + + @Getter @Setter protected String value; + public boolean hasValue () { return !empty(value); } + + @JsonIgnore + public DnsRecordMatch getMatcher() { + return (DnsRecordMatch) new DnsRecordMatch().setFqdn(fqdn).setType(type).setValue(value); + } + +} diff --git a/src/main/java/org/cobbzilla/util/dns/DnsRecordMatch.java b/src/main/java/org/cobbzilla/util/dns/DnsRecordMatch.java new file mode 100644 index 0000000..b7fea14 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/dns/DnsRecordMatch.java @@ -0,0 +1,47 @@ +package org.cobbzilla.util.dns; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.Accessors; + +import java.util.Set; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +@NoArgsConstructor @Accessors(chain=true) @ToString(of={"pattern", "fqdns", "subdomain"}, callSuper=true) +public class DnsRecordMatch extends DnsRecordBase { + + @Getter @Setter private String pattern; + @Getter @Setter private Set fqdns; + public boolean hasFqdns () { return fqdns != null && !fqdns.isEmpty(); } + + public DnsRecordMatch(Set fqdns) { this.fqdns = fqdns; } + + public boolean hasPattern() { return !empty(pattern); } + + @Getter @Setter private String subdomain; + public boolean hasSubdomain() { return !empty(subdomain); } + + public DnsRecordMatch(DnsRecordBase record) { + super(record.getFqdn(), record.getType(), record.getValue()); + } + + public DnsRecordMatch(DnsType type, String fqdn) { + setType(type); + setFqdn(fqdn); + } + + public DnsRecordMatch(String fqdn) { this(null, fqdn); } + + public boolean matches (DnsRecord record) { + if (hasType() && !getType().equals(record.getType())) return false; + if (hasFqdn() && !getFqdn().equals(record.getFqdn())) return false; + if (hasSubdomain() && record.hasFqdn() && !record.getFqdn().endsWith(getSubdomain())) return false; + if (hasPattern() && record.hasFqdn() && !record.getFqdn().matches(getPattern())) return false; + if (hasFqdns() && record.hasFqdn() && !getFqdns().contains(record.getFqdn())) return false; + return true; + } + +} diff --git a/src/main/java/org/cobbzilla/util/dns/DnsServerType.java b/src/main/java/org/cobbzilla/util/dns/DnsServerType.java new file mode 100644 index 0000000..6191403 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/dns/DnsServerType.java @@ -0,0 +1,11 @@ +package org.cobbzilla.util.dns; + +import com.fasterxml.jackson.annotation.JsonCreator; + +public enum DnsServerType { + + dyn, namecheap, djbdns, bind; + + @JsonCreator public static DnsServerType create (String v) { return valueOf(v.toLowerCase()); } + +} diff --git a/src/main/java/org/cobbzilla/util/dns/DnsType.java b/src/main/java/org/cobbzilla/util/dns/DnsType.java new file mode 100644 index 0000000..358ac87 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/dns/DnsType.java @@ -0,0 +1,17 @@ +package org.cobbzilla.util.dns; + +import com.fasterxml.jackson.annotation.JsonCreator; + +public enum DnsType { + + A, AAAA, CNAME, MX, NS, TXT, SOA, PTR, // very common record types + RP, LOC, SIG, SPF, SRV, TSIG, TKEY, CERT, // sometimes used + KEY, DS, DNSKEY, NSEC, NSEC3, NSEC3PARAM, RRSIG, IPSECKEY, DLV, // DNSSEC and other security-related types + DNAME, DLCID, HIP, NAPTR, SSHFP, TLSA, // infrequently used + IXFR, AXFR, OPT; // pseudo-record types + + public static final DnsType[] A_TYPES = new DnsType[] {A, AAAA}; + + @JsonCreator public static DnsType fromString(String value) { return valueOf(value.toUpperCase()); } + +} diff --git a/src/main/java/org/cobbzilla/util/error/GeneralErrorHandler.java b/src/main/java/org/cobbzilla/util/error/GeneralErrorHandler.java new file mode 100644 index 0000000..46a0100 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/error/GeneralErrorHandler.java @@ -0,0 +1,13 @@ +package org.cobbzilla.util.error; + +import org.cobbzilla.util.string.StringUtil; + +import java.util.List; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; + +public interface GeneralErrorHandler { + default T handleError(String message) { return die(message); } + default T handleError(String message, Exception e) { return die(message, e); } + default T handleError(List validationErrors) { return die("validation errors: "+ StringUtil.toString(validationErrors)); } +} diff --git a/src/main/java/org/cobbzilla/util/error/GeneralErrorHandlerBase.java b/src/main/java/org/cobbzilla/util/error/GeneralErrorHandlerBase.java new file mode 100644 index 0000000..b6f1612 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/error/GeneralErrorHandlerBase.java @@ -0,0 +1,11 @@ +package org.cobbzilla.util.error; + +import java.util.concurrent.atomic.AtomicReference; + +public class GeneralErrorHandlerBase implements GeneralErrorHandler { + public static final GeneralErrorHandlerBase instance = new GeneralErrorHandlerBase(); + + public static AtomicReference defaultErrorHandler() { + return new AtomicReference<>(GeneralErrorHandlerBase.instance); + } +} diff --git a/src/main/java/org/cobbzilla/util/error/HasGeneralErrorHandler.java b/src/main/java/org/cobbzilla/util/error/HasGeneralErrorHandler.java new file mode 100644 index 0000000..cbce507 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/error/HasGeneralErrorHandler.java @@ -0,0 +1,12 @@ +package org.cobbzilla.util.error; + +import java.util.concurrent.atomic.AtomicReference; + +public interface HasGeneralErrorHandler { + + AtomicReference getErrorHandler (); + + default T error(String message) { return getErrorHandler().get().handleError(message); } + default T error(String message, Exception e) { return getErrorHandler().get().handleError(message, e); } + +} diff --git a/src/main/java/org/cobbzilla/util/graphics/ColorMode.java b/src/main/java/org/cobbzilla/util/graphics/ColorMode.java new file mode 100644 index 0000000..19957a0 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/graphics/ColorMode.java @@ -0,0 +1,11 @@ +package org.cobbzilla.util.graphics; + +import com.fasterxml.jackson.annotation.JsonCreator; + +public enum ColorMode { + + rgb, ansi; + + @JsonCreator public static ColorMode fromString (String val) { return valueOf(val.toLowerCase()); } + +} diff --git a/src/main/java/org/cobbzilla/util/graphics/ColorUtil.java b/src/main/java/org/cobbzilla/util/graphics/ColorUtil.java new file mode 100644 index 0000000..26733ed --- /dev/null +++ b/src/main/java/org/cobbzilla/util/graphics/ColorUtil.java @@ -0,0 +1,68 @@ +package org.cobbzilla.util.graphics; + +import org.apache.commons.lang3.RandomUtils; + +import java.awt.*; +import java.util.Collection; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.string.StringUtil.getHexValue; + +public class ColorUtil { + + public static final String ANSI_RESET = "\\033[0m"; + + public static int parseRgb(String colorString) { return parseRgb(colorString, null); } + + public static int parseRgb(String colorString, Integer defaultRgb) { + try { + if (empty(colorString)) return defaultRgb; + if (colorString.startsWith("0x")) return Integer.parseInt(colorString.substring(2), 16); + if (colorString.startsWith("#")) return Integer.parseInt(colorString.substring(1), 16); + return Integer.parseInt(colorString, 16); + + } catch (Exception e) { + if (defaultRgb == null) { + return die("parseRgb: '' was unparseable and no default value provided: "+e.getClass().getSimpleName()+": "+e.getMessage(), e); + } + return defaultRgb; + } + } + + public static int rgb2ansi(int color) { return rgb2ansi(new Color(color)); } + + public static int rgb2ansi(Color c) { + return 16 + (36 * (c.getRed() / 51)) + (6 * (c.getGreen() / 51)) + c.getBlue() / 51; + } + + public static String rgb2hex(int color) { + final Color c = new Color(color); + return getHexValue((byte) c.getRed()) + + getHexValue((byte) c.getGreen()) + + getHexValue((byte) c.getBlue()); + } + + public static int randomColor() { return randomColor(null, ColorMode.rgb); } + public static int randomColor(ColorMode mode) { return randomColor(null, mode); } + public static int randomColor(Collection usedColors) { return randomColor(usedColors, ColorMode.rgb); } + + public static int randomColor(Collection usedColors, ColorMode mode) { + int val; + do { + val = RandomUtils.nextInt(0x000000, 0xffffff); + } while (usedColors != null && usedColors.contains(val)); + return mode == ColorMode.rgb ? val : rgb2ansi(val); + } + + public static String ansiColor(int fg) { return ansiColor(fg, null); } + + public static String ansiColor(int fg, Integer bg) { + final StringBuilder b = new StringBuilder(); + b.append("\\033[38;5;") + .append(rgb2ansi(fg)) + .append(bg == null ? "" : ";48;5;"+rgb2ansi(bg)) + .append("m"); + return b.toString(); + } +} diff --git a/src/main/java/org/cobbzilla/util/graphics/ImageTransformConfig.java b/src/main/java/org/cobbzilla/util/graphics/ImageTransformConfig.java new file mode 100644 index 0000000..45f9b17 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/graphics/ImageTransformConfig.java @@ -0,0 +1,23 @@ +package org.cobbzilla.util.graphics; + +import lombok.Getter; +import lombok.Setter; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; + +public class ImageTransformConfig { + + @Getter @Setter private int height; + @Getter @Setter private int width; + + public ImageTransformConfig(String config) { + final int xpos = config.indexOf('x'); + try { + width = Integer.parseInt(config.substring(xpos + 1)); + height = Integer.parseInt(config.substring(0, xpos)); + } catch (Exception e) { + die("invalid config (expected WxH): " + config + ": " + e, e); + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/handlebars/Base64ImageInsertion.java b/src/main/java/org/cobbzilla/util/handlebars/Base64ImageInsertion.java new file mode 100644 index 0000000..f93485d --- /dev/null +++ b/src/main/java/org/cobbzilla/util/handlebars/Base64ImageInsertion.java @@ -0,0 +1,43 @@ +package org.cobbzilla.util.handlebars; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; +import org.apache.commons.codec.binary.Base64InputStream; +import org.cobbzilla.util.io.FileUtil; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.io.FileUtil.temp; + +@NoArgsConstructor @Accessors(chain=true) +public class Base64ImageInsertion extends ImageInsertion { + + public static final Base64ImageInsertion[] NO_IMAGE_INSERTIONS = new Base64ImageInsertion[0]; + + public Base64ImageInsertion(Base64ImageInsertion other) { super(other); } + + public Base64ImageInsertion(String spec) { super(spec); } + + @Getter @Setter private String image; // base64-encoded image data + + @Override public File getImageFile() throws IOException { + if (empty(getImage())) return null; + final File temp = temp("."+getFormat()); + final Base64InputStream stream = new Base64InputStream(new ByteArrayInputStream(image.getBytes())); + FileUtil.toFile(temp, stream); + return temp; + } + + @Override protected void setField(String key, String value) { + switch (key) { + case "image": this.image = value; break; + default: super.setField(key, value); + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/handlebars/ContextMessageSender.java b/src/main/java/org/cobbzilla/util/handlebars/ContextMessageSender.java new file mode 100644 index 0000000..acc852a --- /dev/null +++ b/src/main/java/org/cobbzilla/util/handlebars/ContextMessageSender.java @@ -0,0 +1,7 @@ +package org.cobbzilla.util.handlebars; + +public interface ContextMessageSender { + + void send(String recipient, String subject, String message, String contentType); + +} diff --git a/src/main/java/org/cobbzilla/util/handlebars/ESignInsertion.java b/src/main/java/org/cobbzilla/util/handlebars/ESignInsertion.java new file mode 100644 index 0000000..040853f --- /dev/null +++ b/src/main/java/org/cobbzilla/util/handlebars/ESignInsertion.java @@ -0,0 +1,34 @@ +package org.cobbzilla.util.handlebars; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; + +import java.io.File; + +import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported; + +@NoArgsConstructor @Accessors(chain=true) +public class ESignInsertion extends ImageInsertion { + + public static final ESignInsertion[] NO_ESIGN_INSERTIONS = new ESignInsertion[0]; + + @Getter @Setter private String role = null; + + public ESignInsertion(ESignInsertion other) { super(other); } + + public ESignInsertion(String spec) { super(spec); } + + @Override protected void setField(String key, String value) { + switch (key) { + case "role": role = value; break; + default: super.setField(key, value); + } + } + + @Override public File getImageFile() { + return notSupported("getImageFile not supported for " + this.getClass().getName()); + } + +} diff --git a/src/main/java/org/cobbzilla/util/handlebars/HandlebarsUtil.java b/src/main/java/org/cobbzilla/util/handlebars/HandlebarsUtil.java new file mode 100644 index 0000000..db1997f --- /dev/null +++ b/src/main/java/org/cobbzilla/util/handlebars/HandlebarsUtil.java @@ -0,0 +1,940 @@ +package org.cobbzilla.util.handlebars; + +import com.fasterxml.jackson.core.io.JsonStringEncoder; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.github.jknack.handlebars.Handlebars; +import com.github.jknack.handlebars.HandlebarsException; +import com.github.jknack.handlebars.Helper; +import com.github.jknack.handlebars.Options; +import com.github.jknack.handlebars.io.AbstractTemplateLoader; +import com.github.jknack.handlebars.io.StringTemplateSource; +import com.github.jknack.handlebars.io.TemplateSource; +import lombok.AllArgsConstructor; +import lombok.Cleanup; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.iterators.ArrayIterator; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.pdfbox.io.IOUtils; +import org.cobbzilla.util.collection.SingletonList; +import org.cobbzilla.util.http.HttpContentTypes; +import org.cobbzilla.util.io.FileResolver; +import org.cobbzilla.util.io.FileUtil; +import org.cobbzilla.util.io.PathListFileResolver; +import org.cobbzilla.util.javascript.JsEngineFactory; +import org.cobbzilla.util.reflect.ReflectionUtil; +import org.cobbzilla.util.string.LocaleUtil; +import org.cobbzilla.util.string.StringUtil; +import org.cobbzilla.util.time.JavaTimezone; +import org.cobbzilla.util.time.TimeUtil; +import org.cobbzilla.util.time.UnicodeTimezone; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.Period; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.StringWriter; +import java.lang.reflect.Method; +import java.math.BigDecimal; +import java.math.MathContext; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; + +import static java.util.regex.Pattern.quote; +import static org.cobbzilla.util.collection.ArrayUtil.arrayToString; +import static org.cobbzilla.util.daemon.DaemonThreadFactory.fixedPool; +import static org.cobbzilla.util.daemon.ZillaRuntime.*; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.io.FileUtil.touch; +import static org.cobbzilla.util.io.StreamUtil.loadResourceAsStream; +import static org.cobbzilla.util.io.StreamUtil.stream2string; +import static org.cobbzilla.util.json.JsonUtil.getJsonStringEncoder; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.security.ShaUtil.sha256_hex; +import static org.cobbzilla.util.string.Base64.encodeBytes; +import static org.cobbzilla.util.string.Base64.encodeFromFile; +import static org.cobbzilla.util.string.StringUtil.*; +import static org.cobbzilla.util.system.CommandShell.*; + +@AllArgsConstructor @Slf4j +public class HandlebarsUtil extends AbstractTemplateLoader { + + public static final char HB_START_CHAR = '{'; + public static final char HB_END_CHAR = '}'; + + public static final String HB_START = StringUtils.repeat(HB_START_CHAR, 2); + public static final String HB_END = StringUtils.repeat(HB_END_CHAR, 2); + + public static final String HB_LSTART = StringUtils.repeat(HB_START_CHAR, 3); + public static final String HB_LEND = StringUtils.repeat(HB_END_CHAR, 3); + + public static final String DEFAULT_FLOAT_FORMAT = "%1$,.3f"; + public static final JsonStringEncoder JSON_STRING_ENCODER = new JsonStringEncoder(); + + private String sourceName = "unknown"; + + public static Map apply(Handlebars handlebars, Map map, Map ctx) { + return apply(handlebars, map, ctx, HB_START_CHAR, HB_END_CHAR); + } + + public static Map apply(Handlebars handlebars, Map map, Map ctx, char altStart, char altEnd) { + if (empty(map)) return map; + final Map merged = new LinkedHashMap<>(); + final String hbStart = StringUtils.repeat(altStart, 2); + final String hbEnd = StringUtils.repeat(altEnd, 2); + for (Map.Entry entry : map.entrySet()) { + final Object value = entry.getValue(); + if (value instanceof String) { + final String val = (String) value; + if (val.contains(hbStart) && val.contains(hbEnd)) { + merged.put(entry.getKey(), apply(handlebars, value.toString(), ctx, altStart, altEnd)); + } else { + merged.put(entry.getKey(), entry.getValue()); + } + + } else if (value instanceof Map) { + // recurse + merged.put(entry.getKey(), apply(handlebars, (Map) value, ctx, altStart, altEnd)); + + } else { + log.info("apply: "); + merged.put(entry.getKey(), entry.getValue()); + } + } + return merged; + } + + public static String apply(Handlebars handlebars, String value, Map ctx) { + return apply(handlebars, value, ctx, (char) 0, (char) 0); + } + + public static final String DUMMY_START3 = "~~~___~~~"; + public static final String DUMMY_START2 = "~~__~~"; + public static final String DUMMY_END3 = "%%%\\^\\^\\^%%%"; + public static final String DUMMY_END2 = "\\$\\$\\^\\^\\$\\$"; + public static String apply(Handlebars handlebars, String value, Map ctx, char altStart, char altEnd) { + if (value == null) return null; + if (altStart != 0 && altEnd != 0 && (altStart != HB_START_CHAR && altEnd != HB_END_CHAR)) { + final String s3 = StringUtils.repeat(altStart, 3); + final String s2 = StringUtils.repeat(altStart, 2); + final String e3 = StringUtils.repeat(altEnd, 3); + final String e2 = StringUtils.repeat(altEnd, 2); + // escape existing handlebars delimiters with dummy placeholders (we'll put them back later) + value = value.replaceAll(quote(HB_LSTART), DUMMY_START3).replaceAll(HB_LEND, DUMMY_END3) + .replaceAll(quote(HB_START), DUMMY_START2).replaceAll(HB_END, DUMMY_END2) + // replace our custom start/end delimiters with handlebars standard ones + .replaceAll(quote(s3), HB_LSTART).replaceAll(quote(e3), HB_LEND) + .replaceAll(quote(s2), HB_START).replaceAll(quote(e2), HB_END); + // run handlebars, then put the real handlebars stuff back (removing the dummy placeholders) + value = apply(handlebars, value, ctx) + .replaceAll(DUMMY_START3, HB_LSTART).replaceAll(DUMMY_END3, HB_LEND) + .replaceAll(DUMMY_START2, HandlebarsUtil.HB_START).replaceAll(DUMMY_END2, HB_END); + return value; + } + try { + @Cleanup final StringWriter writer = new StringWriter(value.length()); + handlebars.compile(value).apply(ctx, writer); + return writer.toString(); + } catch (HandlebarsException e) { + final Throwable cause = e.getCause(); + if (cause != null && ((cause instanceof FileNotFoundException) || (cause instanceof RequiredVariableUndefinedException))) { + log.error(e.getMessage()+": \""+value+"\""); + throw e; + } + return die("apply("+value+"): "+e, e); + } catch (Exception e) { + return die("apply("+value+"): "+e, e); + } catch (Error e) { + log.warn("apply: "+e, e); + throw e; + } + } + + /** + * Using reflection, we find all public getters of a thing (and if the getter returns an object, find all + * of its public getters, recursively and so on). We limit our results to those getters that have corresponding + * setters: methods whose sole parameter is of a compatible type with the return type of the getter. + * For each such property whose value is a String, we apply handlebars using the provided context. + * @param handlebars the handlebars template processor + * @param thing the object to operate upon + * @param ctx the context to apply + * @param the return type + * @return the thing, possibly with String-valued properties having been modified + */ + public static T applyReflectively(Handlebars handlebars, T thing, Map ctx) { + return applyReflectively(handlebars, thing, ctx, HB_START_CHAR, HB_END_CHAR); + } + + public static T applyReflectively(Handlebars handlebars, T thing, Map ctx, char altStart, char altEnd) { + for (Method getterCandidate : thing.getClass().getMethods()) { + + if (!getterCandidate.getName().startsWith("get")) continue; + if (!canApplyReflectively(getterCandidate.getReturnType())) continue; + + final String setterName = ReflectionUtil.setterForGetter(getterCandidate.getName()); + for (Method setterCandidate : thing.getClass().getMethods()) { + if (!setterCandidate.getName().equals(setterName) + || setterCandidate.getParameterTypes().length != 1 + || !setterCandidate.getParameterTypes()[0].isAssignableFrom(getterCandidate.getReturnType())) { + continue; + } + try { + final Object value = getterCandidate.invoke(thing, (Object[]) null); + if (value == null) break; + if (value instanceof String) { + if (value.toString().contains("" + altStart + altStart)) { + setterCandidate.invoke(thing, apply(handlebars, (String) value, ctx, altStart, altEnd)); + } + + } else if (value instanceof JsonNode) { + setterCandidate.invoke(thing, json(apply(handlebars, json(value), ctx, altStart, altEnd), JsonNode.class)); + + } else if (value instanceof Map) { + setterCandidate.invoke(thing, apply(handlebars, (Map) value, ctx, altStart, altEnd)); + +// } else if (Object[].class.isAssignableFrom(value.getClass())) { +// final Object[] array = (Object[]) value; +// final Object[] rendered = new Object[array.length]; +// for (int i=0; i returnType) { + if (returnType.equals(String.class)) return true; + try { + return !(returnType.isPrimitive() || (returnType.getPackage() != null && returnType.getPackage().getName().equals("java.lang"))); + } catch (NullPointerException npe) { + log.warn("canApplyReflectively("+returnType+"): "+npe); + return false; + } + } + + @Override public TemplateSource sourceAt(String source) throws IOException { + return new StringTemplateSource(sourceName, source); + } + + public static final CharSequence EMPTY_SAFE_STRING = ""; + + private static final AtomicReference messageSender = new AtomicReference<>(); + + public static void setMessageSender(ContextMessageSender sender) { + synchronized (messageSender) { + final ContextMessageSender current = messageSender.get(); + if (current != null && current != sender && !current.equals(sender)) die("setMessageSender: already set to "+current); + messageSender.set(sender); + } + } + + public static void registerUtilityHelpers (final Handlebars hb) { + hb.registerHelper("hostname", (src, options) -> { + switch (src.toString()) { + case "short": return new Handlebars.SafeString(hostname_short()); + case "domain": return new Handlebars.SafeString(domainname()); + case "regular": default: return new Handlebars.SafeString(hostname()); + } + }); + + hb.registerHelper("exists", (src, options) -> empty(src) ? null : options.apply(options.fn)); + + hb.registerHelper("not_exists", (src, options) -> !empty(src) ? null : options.apply(options.fn)); + + hb.registerHelper("sha256", (src, options) -> { + if (empty(src)) return ""; + src = apply(hb, src.toString(), (Map) options.context.model()); + src = sha256_hex(src.toString()); + return new Handlebars.SafeString(src.toString()); + }); + + hb.registerHelper("format_epoch", (val, options) -> { + if (empty(val)) return ""; + if (options.params.length != 2) return die("format_epoch: Usage: {{format_epoch expr format timezone}}"); + final String format = options.param(0); + final String timezone = options.param(1); + + DateTimeZone tz; + try { + final JavaTimezone jtz = JavaTimezone.fromString(timezone); + if (jtz == null) { + final UnicodeTimezone utz = UnicodeTimezone.fromString(timezone); + if (utz == null) return die("format_epoch: invalid timezone: "+timezone); + tz = DateTimeZone.forTimeZone(utz.toJava().getTimeZone()); + } else { + tz = DateTimeZone.forTimeZone(jtz.getTimeZone()); + } + } catch (Exception e) { + log.warn("format_epoch: timezone error: "+timezone+", will retry with default parsing, or use GMT: "+e); + tz = DateTimeZone.forTimeZone(TimeZone.getTimeZone(timezone)); + } + + return new Handlebars.SafeString(DateTimeFormat.forPattern(format).withZone(tz).print(Long.valueOf(val.toString().trim()))); + }); + + hb.registerHelper("format_float", (val, options) -> { + if (empty(val)) return ""; + if (options.params.length > 2) return die("format_float: too many parameters. Usage: {{format_float expr [format] [locale]}}"); + final String format = options.params.length > 0 && !empty(options.param(0)) ? options.param(0) : DEFAULT_FLOAT_FORMAT; + final Locale locale = LocaleUtil.fromString(options.params.length > 1 && !empty(options.param(1)) ? options.param(1) : null); + + val = apply(hb, val.toString(), (Map) options.context.model()); + val = String.format(locale, format, Double.valueOf(val.toString())); + return new Handlebars.SafeString(val.toString()); + }); + + hb.registerHelper("json", (src, options) -> { + if (empty(src)) return ""; + return new Handlebars.SafeString(json(src)); + }); + + hb.registerHelper("escaped_json", (src, options) -> { + if (empty(src)) return ""; + return new Handlebars.SafeString(new String(JSON_STRING_ENCODER.quoteAsString(json(src)))); + }); + + hb.registerHelper("escaped_regex", (src, options) -> { + if (empty(src)) return ""; + return new Handlebars.SafeString(Pattern.quote(src.toString())); + }); + + hb.registerHelper("url_encoded_escaped_regex", (src, options) -> { + if (empty(src)) return ""; + return new Handlebars.SafeString(urlEncode(Pattern.quote(src.toString()))); + }); + + hb.registerHelper("escape_js_single_quoted_string", (src, options) -> { + if (empty(src)) return ""; + return new Handlebars.SafeString(src.toString().replace("'", "\\'")); + }); + + hb.registerHelper("context", (src, options) -> { + if (empty(src)) return ""; + if (options.params.length > 0) return die("context: too many parameters. Usage: {{context [recipient]}}"); + final String ctxString = options.context.toString(); + final String recipient = src.toString(); + final String subject = options.params.length > 1 ? options.param(0) : null; + sendContext(recipient, subject, ctxString,HttpContentTypes.TEXT_PLAIN); + return new Handlebars.SafeString(ctxString); + }); + + hb.registerHelper("context_json", (src, options) -> { + if (empty(src)) return ""; + try { + if (options.params.length > 0) return die("context: too many parameters. Usage: {{context [recipient]}}"); + final String json = json(options.context.model()); + final String recipient = src.toString(); + final String subject = options.params.length > 1 ? options.param(0) : null; + sendContext(recipient, subject, json, HttpContentTypes.APPLICATION_JSON); + return new Handlebars.SafeString(json); + } catch (Exception e) { + return new Handlebars.SafeString("Error calling json(options.context): "+e.getClass()+": "+e.getMessage()); + } + }); + + hb.registerHelper("required", (src, options) -> { + if (src == null) throw new RequiredVariableUndefinedException("required: undefined variable"); + return new Handlebars.SafeString(src.toString()); + }); + + hb.registerHelper("safe_name", (src, options) -> { + if (empty(src)) return ""; + return new Handlebars.SafeString(safeSnakeName(src.toString())); + }); + + hb.registerHelper("urlEncode", (src, options) -> { + if (empty(src)) return ""; + src = apply(hb, src.toString(), (Map) options.context.model()); + src = urlEncode(src.toString()); + return new Handlebars.SafeString(src.toString()); + }); + + hb.registerHelper("lastElement", (thing, options) -> { + if (thing == null) return null; + final Iterator iter = getIterator(thing); + final String path = options.param(0); + Object lastElement = null; + while (iter.hasNext()) { + lastElement = iter.next(); + } + final Object val = ReflectionUtil.get(lastElement, path); + if (val != null) return new Handlebars.SafeString(""+val); + return EMPTY_SAFE_STRING; + }); + + hb.registerHelper("find", (thing, options) -> { + if (thing == null) return null; + final Iterator iter = getIterator(thing); + final String path = options.param(0); + final String arg = options.param(1); + final String output = options.param(2); + while (iter.hasNext()) { + final Object item = iter.next(); + try { + final Object val = ReflectionUtil.get(item, path); + if (val != null && String.valueOf(val).equals(arg)) { + return new Handlebars.SafeString(""+ReflectionUtil.get(item, output)); + } + } catch (Exception e) { + log.warn("find: "+e); + } + } + return EMPTY_SAFE_STRING; + }); + + hb.registerHelper("compare", (val1, options) -> { + final String operator = options.param(0); + final Object val2 = getComparisonArgParam(options); + final Comparable v1 = cval(val1); + final Object v2 = cval(val2); + return (v1 == null && v2 == null) || (v1 != null && compare(operator, v1, v2)) ? options.fn(options) : options.inverse(options); + }); + + hb.registerHelper("not", (val1, options) -> + new Handlebars.SafeString(""+(!Boolean.valueOf(val1 == null ? "false" : val1.toString()))) + ); + + hb.registerHelper("string_compare", (val1, options) -> { + final String operator = options.param(0); + final Object val2 = getComparisonArgParam(options); + final String v1 = val1 == null ? null : val1.toString(); + final String v2 = val2 == null ? null : val2.toString(); + return compare(operator, v1, v2) ? options.fn(options) : options.inverse(options); + }); + + hb.registerHelper("long_compare", (val1, options) -> { + final String operator = options.param(0); + final Object val2 = getComparisonArgParam(options); + final Long v1 = val1 == null ? null : Long.valueOf(val1.toString()); + final Long v2 = val2 == null ? null : Long.valueOf(val2.toString()); + return compare(operator, v1, v2) ? options.fn(options) : options.inverse(options); + }); + + hb.registerHelper("double_compare", (val1, options) -> { + final String operator = options.param(0); + final Object val2 = getComparisonArgParam(options); + final Double v1 = val1 == null ? null : Double.valueOf(val1.toString()); + final Double v2 = val2 == null ? null : Double.valueOf(val2.toString()); + return compare(operator, v1, v2) ? options.fn(options) : options.inverse(options); + }); + + hb.registerHelper("big_compare", (val1, options) -> { + final String operator = options.param(0); + final Object val2 = getComparisonArgParam(options); + final BigDecimal v1 = val1 == null ? null : big(val1.toString()); + final BigDecimal v2 = val2 == null ? null : big(val2.toString()); + return compare(operator, v1, v2) ? options.fn(options) : options.inverse(options); + }); + + hb.registerHelper("expr", (val1, options) -> { + final String operator = options.param(0); + final String format = options.params.length > 2 ? options.param(2) : null; + final Object val2 = getComparisonArgParam(options); + final String v1 = val1.toString(); + final String v2 = val2.toString(); + + final BigDecimal result; + switch (operator) { + case "+": result = big(v1).add(big(v2)); break; + case "-": result = big(v1).subtract(big(v2)); break; + case "*": result = big(v1).multiply(big(v2)); break; + case "/": result = big(v1).divide(big(v2), MathContext.DECIMAL128); break; + case "%": result = big(v1).remainder(big(v2)).abs(); break; + case "^": result = big(v1).pow(big(v2).intValue()); break; + default: return die("expr: invalid operator: "+operator); + } + + // can't use trigraph (?:) operator here, if we do then for some reason rval always ends up as a double + final Number rval; + if (v1.contains(".") || v2.contains(".") || operator.equals("/")) { + rval = result.doubleValue(); + } else { + rval = result.intValue(); + } + if (format != null) { + final Locale locale = LocaleUtil.fromString(options.params.length > 3 && !empty(options.param(3)) ? options.param(3) : null); + return new Handlebars.SafeString(String.format(locale, format, rval)); + } else { + return new Handlebars.SafeString(rval.toString()); + } + }); + + hb.registerHelper("rand", (Helper) (len, options) -> { + final String kind = options.param(0, "alphanumeric"); + final String alphaCase = options.param(1, "lowercase"); + switch (kind) { + case "alphanumeric": case "alnum": default: return new Handlebars.SafeString(adjustCase(RandomStringUtils.randomAlphanumeric(len), alphaCase)); + case "alpha": case "alphabetic": return new Handlebars.SafeString(adjustCase(RandomStringUtils.randomAlphabetic(len), alphaCase)); + case "num": case "numeric": return new Handlebars.SafeString(RandomStringUtils.randomNumeric(len)); + } + }); + + hb.registerHelper("truncate", (Helper) (max, options) -> { + final String val = options.param(0, " "); + if (empty(val)) return ""; + if (max == -1 || max >= val.length()) return val; + return new Handlebars.SafeString(val.substring(0, max)); + }); + + hb.registerHelper("truncate_and_url_encode", (Helper) (max, options) -> { + final String val = options.param(0, " "); + if (empty(val)) return ""; + if (max == -1 || max >= val.length()) return simpleUrlEncode(val); + return new Handlebars.SafeString(simpleUrlEncode(val.substring(0, max))); + }); + + hb.registerHelper("truncate_and_double_url_encode", (Helper) (max, options) -> { + final String val = options.param(0, " "); + if (empty(val)) return ""; + if (max == -1 || max >= val.length()) return simpleUrlEncode(simpleUrlEncode(val)); + return new Handlebars.SafeString(simpleUrlEncode(simpleUrlEncode(val.substring(0, max)))); + }); + + hb.registerHelper("length", (thing, options) -> { + if (empty(thing)) return "0"; + if (thing.getClass().isArray()) return ""+((Object[]) thing).length; + if (thing instanceof Collection) return ""+((Collection) thing).size(); + if (thing instanceof ArrayNode) return ""+((ArrayNode) thing).size(); + return ""; + }); + + hb.registerHelper("first_nonempty", (thing, options) -> { + if (!empty(thing)) return new Handlebars.SafeString(thing.toString()); + for (final Object param : options.params) { + if (!empty(param)) return new Handlebars.SafeString(param.toString()); + } + return EMPTY_SAFE_STRING; + }); + + hb.registerHelper("key_file", (thing, options) -> { + if (empty(thing)) return die("key_file: no file provided"); + try { + String path = thing.toString(); + final String[] paths = new String[]{ + System.getProperty("user.home") + "/" + path, + System.getProperty("user.dir") + "/" + path, + path + }; + log.debug("key_file: checking paths: "+arrayToString(paths, ",")); + File found = null; + File okToCreate = null; + for (String p : paths) { + try { + final File f = new File(p); + if (f.exists() && f.canRead() && f.length() > 0) { + log.debug("key_file: assigning found="+abs(f)); + found = f; + break; + } + if (f.getParentFile().canWrite()) { + log.debug("key_file: assigning okToCreate="+abs(f)); + okToCreate = f; + break; + } + } catch (Exception e) { + log.warn("key_file: "+e.getClass().getName()+": "+e.getMessage()); + } + } + if (found != null) { + log.debug("key_file: returning contents of found: "+abs(found)); + chmod(found, "o-rwx"); + return new Handlebars.SafeString(FileUtil.toString(found).trim()); + } + if (okToCreate != null) { + touch(okToCreate); + initKey(okToCreate); + chmod(okToCreate, "o-rwx"); + log.info("key_file: chmodding and returning contents of okToCreate: "+abs(okToCreate)); + return new Handlebars.SafeString(FileUtil.toString(okToCreate)); + } + return die("key_file: no file could be found or created"); + } catch (Exception e) { + return die("key_file: "+e, e); + } + }); + } + + private static String adjustCase(String val, String alphaCase) { + switch (alphaCase) { + case "lower": case "lowercase": case "lc": return val.toLowerCase(); + case "upper": case "uppercase": case "uc": return val.toUpperCase(); + default: return val; + } + } + + private static String initKey(File f) throws IOException { + chmod(f, "600"); + FileUtil.toFile(f, UUID.randomUUID().toString()); + return FileUtil.toString(f); + } + + public static Object getComparisonArgParam(Options options) { + if (options.params.length <= 1) return die("getComparisonArgParam: missing argument"); + return options.param(1); + } + + public static String getEmailRecipient(Handlebars hb, Options options, int index) { + return options.params.length > index && !empty(options.param(index)) + ? apply(hb, options.param(index).toString(), (Map) options.context.model()) + : null; + } + + private static final ExecutorService contextSender = fixedPool(10); + + public static void sendContext(String recipient, String subject, String message, String contentType) { + contextSender.submit(() -> { + if (!empty(recipient) && !empty(message)) { + synchronized (messageSender) { + final ContextMessageSender sender = messageSender.get(); + if (sender != null) { + try { + sender.send(recipient, subject, message, contentType); + } catch (Exception e) { + log.error("context: error sending message: " + e, e); + } + } + } + } + }); + } + + private static Iterator getIterator(Object thing) { + if (thing instanceof Collection) { + return ((Collection) thing).iterator(); + } else if (thing instanceof Map) { + return ((Map) thing).values().iterator(); + } else if (Object[].class.isAssignableFrom(thing.getClass())) { + return new ArrayIterator(thing); + } else { + return die("find: invalid argument type "+thing.getClass().getName()); + } + } + + private static Comparable cval(Object v) { + if (v == null) return null; + if (v instanceof Number) return (Comparable) v; + if (v instanceof String) { + final String s = v.toString(); + try { + return Long.parseLong(s); + } catch (Exception e) { + try { + return big(s); + } catch (Exception e2) { + return s; + } + } + } else { + return die("don't know to compare objects of class "+v.getClass()); + } + } + + public static boolean compare(String operator, Comparable v1, T v2) { + if (v1 == null) return v2 == null; + if (v2 == null) return false; + boolean result; + final List parts; + switch (operator) { + case "==": result = v1.equals(v2); break; + case "!=": result = !v1.equals(v2); break; + case ">": result = v1.compareTo(v2) > 0; break; + case ">=": result = v1.compareTo(v2) >= 0; break; + case "<": result = v1.compareTo(v2) < 0; break; + case "<=": result = v1.compareTo(v2) <= 0; break; + case "in": + parts = StringUtil.split(v2.toString(), ", \n\t"); + for (String part : parts) { + if (v1.equals(part)) return true; + } + return false; + case "not_in": + parts = StringUtil.split(v2.toString(), ", \n\t"); + for (String part : parts) { + if (v1.equals(part)) return false; + } + return true; + default: result = false; + } + return result; + } + + public static void registerCurrencyHelpers(Handlebars hb) { + hb.registerHelper("dollarsNoSign", (src, options) -> { + if (empty(src)) return ""; + return new Handlebars.SafeString(formatDollarsNoSign(longDollarVal(src))); + }); + + hb.registerHelper("dollarsWithSign", (src, options) -> { + if (empty(src)) return ""; + return new Handlebars.SafeString(formatDollarsWithSign(longDollarVal(src))); + }); + + hb.registerHelper("dollarsAndCentsNoSign", (src, options) -> { + if (empty(src)) return ""; + return new Handlebars.SafeString(formatDollarsAndCentsNoSign(longDollarVal(src))); + }); + + hb.registerHelper("dollarsAndCentsWithSign", (src, options) -> { + if (empty(src)) return ""; + return new Handlebars.SafeString(formatDollarsAndCentsWithSign(longDollarVal(src))); + }); + + hb.registerHelper("dollarsAndCentsPlain", (src, options) -> { + if (empty(src)) return ""; + return new Handlebars.SafeString(formatDollarsAndCentsPlain(longDollarVal(src))); + }); + } + + @Getter @Setter private static String defaultTimeZone = "US/Eastern"; + + private abstract static class DateHelper implements Helper { + + protected DateTimeZone getTimeZone (Options options) { return getTimeZone(options, 0); } + + protected DateTimeZone getTimeZone (Options options, int index) { + final String timeZoneName = options.param(index, getDefaultTimeZone()); + try { + return DateTimeZone.forID(timeZoneName); + } catch (Exception e) { + return die("getTimeZone: invalid timezone: "+timeZoneName); + } + } + + protected long zonedTimestamp (Object src, Options options) { return zonedTimestamp(src, options, 0); } + + protected long zonedTimestamp (Object src, Options options, int index) { + if (empty(src)) src = "now"; + final DateTimeZone timeZone = getTimeZone(options, index); + return longVal(src, timeZone); + } + + protected CharSequence print (DateTimeFormatter formatter, Object src, Options options) { + return new Handlebars.SafeString(formatter.print(new DateTime(zonedTimestamp(src, options), getTimeZone(options)))); + } + } + + public static void registerDateHelpers(Handlebars hb) { + + hb.registerHelper("date_format", new DateHelper() { + public CharSequence apply(Object src, Options options) { + final DateTimeFormatter formatter = DateTimeFormat.forPattern(options.param(0)); + return new Handlebars.SafeString(formatter.print(new DateTime(zonedTimestamp(src, options, 1), + getTimeZone(options, 1)))); + } + }); + + hb.registerHelper("date_short", new DateHelper() { + public CharSequence apply(Object src, Options options) { + return print(TimeUtil.DATE_FORMAT_MMDDYYYY, src, options); + } + }); + + hb.registerHelper("date_yyyy_mm_dd", new DateHelper() { + public CharSequence apply(Object src, Options options) { + return print(TimeUtil.DATE_FORMAT_YYYY_MM_DD, src, options); + } + }); + + hb.registerHelper("date_mmm_dd_yyyy", new DateHelper() { + public CharSequence apply(Object src, Options options) { + return print(TimeUtil.DATE_FORMAT_MMM_DD_YYYY, src, options); + } + }); + + hb.registerHelper("date_long", new DateHelper() { + public CharSequence apply(Object src, Options options) { + return print(TimeUtil.DATE_FORMAT_MMMM_D_YYYY, src, options); + } + }); + + hb.registerHelper("timestamp", new DateHelper() { + public CharSequence apply(Object src, Options options) { + return new Handlebars.SafeString(Long.toString(zonedTimestamp(src, options))); + } + }); + } + + private static long longVal(Object src, DateTimeZone timeZone) { + return longVal(src, timeZone, true); + } + + private static long longVal(Object src, DateTimeZone timeZone, boolean tryAgain) { + if (src == null) return now(); + String srcStr = src.toString().trim(); + + if (srcStr.equals("") || srcStr.equals("0") || srcStr.equals("now")) return now(); + + if (srcStr.startsWith("now")) { + // Multiple periods may be added to the original timestamp (separated by comma), but in the correct order. + final String[] splitSrc = srcStr.substring(3).split(","); + DateTime result = new DateTime(now(), timeZone).withTimeAtStartOfDay(); + for (String period : splitSrc) { + result = result.plus(Period.parse(period, TimeUtil.PERIOD_FORMATTER)); + } + return result.getMillis(); + } + + try { + return ((Number) src).longValue(); + } catch (Exception e) { + if (!tryAgain) return die("longVal: unparseable long: "+src+": "+e.getClass().getSimpleName()+": "+e.getMessage()); + + // try to parse it in different formats + final Object t = TimeUtil.parse(src.toString(), timeZone); + return longVal(t, timeZone, false); + } + } + + public static long longDollarVal(Object src) { + final Long val = ReflectionUtil.toLong(src); + return val == null ? 0 : val; + } + + public static final String CLOSE_XML_DECL = "?>"; + + public static void registerXmlHelpers(final Handlebars hb) { + hb.registerHelper("strip_xml_declaration", (src, options) -> { + if (empty(src)) return ""; + String xml = src.toString().trim(); + if (xml.startsWith(" { + if (empty(src)) return ""; + return new Handlebars.SafeString(jurisdictionResolver.usState(src.toString())); + }); + hb.registerHelper("us_zip", (src, options) -> { + if (empty(src)) return ""; + return new Handlebars.SafeString(jurisdictionResolver.usZip(src.toString())); + }); + } + + public static void registerJavaScriptHelper(final Handlebars hb, JsEngineFactory jsEngineFactory) { + hb.registerHelper("js", (src, options) -> { + if (empty(src)) return ""; + + final String format = options.params.length > 0 && !empty(options.param(0)) ? options.param(0) : null; + final Locale locale = LocaleUtil.fromString(options.params.length > 1 && !empty(options.param(1)) ? options.param(1) : null); + + final Map ctx = (Map) options.context.model(); + final Object result = jsEngineFactory.getJs().evaluate(src.toString(), ctx); + if (result == null) return new Handlebars.SafeString("null"); + return format != null + ? new Handlebars.SafeString(String.format(locale, format, Double.valueOf(result.toString()))) + : new Handlebars.SafeString(result.toString()); + }); + } + + public static final String DEFAULT_FILE_RESOLVER = "_"; + private static final Map fileResolverMap = new HashMap<>(); + + public static void setFileIncludePath(String path) { setFileIncludePaths(DEFAULT_FILE_RESOLVER, new SingletonList<>(path)); } + + public static void setFileIncludePaths(Collection paths) { setFileIncludePaths(DEFAULT_FILE_RESOLVER, paths); } + + public static void setFileIncludePaths(String name, Collection paths) { + fileResolverMap.put(name, new PathListFileResolver(paths)); + } + + @AllArgsConstructor + private static class FileLoaderHelper implements Helper { + + private boolean isBase64EncoderOn; + + @Override public CharSequence apply(String filename, Options options) throws IOException { + if (empty(filename)) return EMPTY_SAFE_STRING; + + final String include = options.get("includePath", DEFAULT_FILE_RESOLVER); + final FileResolver fileResolver = fileResolverMap.get(include); + if (fileResolver == null) return die("apply: no file resolve found for includePath="+include); + + final boolean escapeSpecialChars = options.get("escape", false); + + File f = fileResolver.resolve(filename); + if (f == null && filename.startsWith(File.separator)) { + // looks like an absolute path, try the filesystem + f = new File(filename); + if (!f.exists() || !f.canRead()) f = null; + } + + if (f == null) { + // try classpath + try { + String content = isBase64EncoderOn + ? encodeBytes(IOUtils.toByteArray(loadResourceAsStream(filename))) + : stream2string(filename); + if (escapeSpecialChars) { + content = new String(getJsonStringEncoder().quoteAsString(content)); + } + return new Handlebars.SafeString(content); + } catch (Exception e) { + throw new FileNotFoundException("Cannot find readable file " + filename + ", resolver: " + fileResolver); + } + } + + try { + String string = isBase64EncoderOn ? encodeFromFile(f) : FileUtil.toString(f); + if (escapeSpecialChars) string = new String(getJsonStringEncoder().quoteAsString(string)); + return new Handlebars.SafeString(string); + } catch (IOException e) { + return die("Cannot read file from: " + f, e); + } + } + } + + public static void registerFileHelpers(final Handlebars hb) { + hb.registerHelper("rawImagePng", (src, options) -> { + if (empty(src)) return ""; + + final String include = options.get("includePath", DEFAULT_FILE_RESOLVER); + final FileResolver fileResolver = fileResolverMap.get(include); + if (fileResolver == null) return die("rawImagePng: no file resolve found for includePath="+include); + + final File f = fileResolver.resolve(src.toString()); + String imgSrc = (f == null) ? src.toString() : f.getAbsolutePath(); + + final Object width = options.get("width"); + final String widthAttr = empty(width) ? "" : "width=\"" + width + "\" "; + return new Handlebars.SafeString( + ""); + }); + + hb.registerHelper("base64File", new FileLoaderHelper(true)); + hb.registerHelper("textFile", new FileLoaderHelper(false)); + } +} diff --git a/src/main/java/org/cobbzilla/util/handlebars/HasHandlebars.java b/src/main/java/org/cobbzilla/util/handlebars/HasHandlebars.java new file mode 100644 index 0000000..5cc85b7 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/handlebars/HasHandlebars.java @@ -0,0 +1,9 @@ +package org.cobbzilla.util.handlebars; + +import com.github.jknack.handlebars.Handlebars; + +public interface HasHandlebars { + + Handlebars getHandlebars(); + +} diff --git a/src/main/java/org/cobbzilla/util/handlebars/ImageInsertion.java b/src/main/java/org/cobbzilla/util/handlebars/ImageInsertion.java new file mode 100644 index 0000000..add4330 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/handlebars/ImageInsertion.java @@ -0,0 +1,63 @@ +package org.cobbzilla.util.handlebars; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; +import org.cobbzilla.util.string.StringUtil; + +import java.io.File; +import java.io.IOException; +import java.util.Map; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.reflect.ReflectionUtil.copy; + +@NoArgsConstructor @Accessors(chain=true) +public abstract class ImageInsertion { + + @Getter @Setter private String name = null; + @Getter @Setter private int page = 0; + @Getter @Setter private float x; + @Getter @Setter private float y; + @Getter @Setter private float width = 0; + @Getter @Setter private float height = 0; + @Getter @Setter private String format = "png"; + @Getter @Setter private boolean watermark = false; + + @JsonIgnore public abstract File getImageFile() throws IOException; + + public ImageInsertion(ImageInsertion other) { copy(this, other); } + + public ImageInsertion(String spec) { + for (String part : StringUtil.split(spec, ", ")) { + final int eqPos = part.indexOf("="); + if (eqPos == -1) die("invalid image insertion (missing '='): "+spec); + if (eqPos == part.length()-1) die("invalid image insertion (no value): "+spec); + final String key = part.substring(0, eqPos).trim(); + final String value = part.substring(eqPos+1).trim(); + setField(key, value); + } + } + + public void init (Map map) { + for (Map.Entry entry : map.entrySet()) { + setField(entry.getKey(), entry.getValue().toString()); + } + } + + protected void setField(String key, String value) { + switch (key) { + case "name": this.name = value; break; + case "page": this.page = Integer.parseInt(value); break; + case "x": this.x = Float.parseFloat(value); break; + case "y": this.y = Float.parseFloat(value); break; + case "width": this.width = Float.parseFloat(value); break; + case "height": this.height = Float.parseFloat(value); break; + case "format": this.format = value; break; + default: die("invalid parameter: "+key); + } + + } +} diff --git a/src/main/java/org/cobbzilla/util/handlebars/JurisdictionResolver.java b/src/main/java/org/cobbzilla/util/handlebars/JurisdictionResolver.java new file mode 100644 index 0000000..bc418eb --- /dev/null +++ b/src/main/java/org/cobbzilla/util/handlebars/JurisdictionResolver.java @@ -0,0 +1,13 @@ +package org.cobbzilla.util.handlebars; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +public interface JurisdictionResolver { + + String usState (String value); + + String usZip (String value); + + default boolean isValidUsStateAbbreviation(String a) { return !empty(a) && usState(a) != null; } + +} diff --git a/src/main/java/org/cobbzilla/util/handlebars/PdfMerger.java b/src/main/java/org/cobbzilla/util/handlebars/PdfMerger.java new file mode 100644 index 0000000..75408e7 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/handlebars/PdfMerger.java @@ -0,0 +1,251 @@ +package org.cobbzilla.util.handlebars; + +import com.github.jknack.handlebars.Handlebars; +import lombok.Cleanup; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.pdfbox.io.MemoryUsageSetting; +import org.apache.pdfbox.multipdf.PDFMergerUtility; +import org.apache.pdfbox.pdmodel.*; +import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject; +import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm; +import org.apache.pdfbox.pdmodel.interactive.form.PDCheckBox; +import org.apache.pdfbox.pdmodel.interactive.form.PDField; +import org.apache.pdfbox.pdmodel.interactive.form.PDTextField; +import org.cobbzilla.util.error.GeneralErrorHandler; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.error.GeneralErrorHandlerBase.defaultErrorHandler; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.io.FileUtil.temp; +import static org.cobbzilla.util.reflect.ReflectionUtil.instantiate; + +@Slf4j +public class PdfMerger { + + @Getter private static final AtomicReference errorHandler = defaultErrorHandler(); + public static void setErrorHandler (GeneralErrorHandler handler) { errorHandler.set(handler); } + + public static final String NULL_FORM_VALUE = "þÿ"; + public static final String CTX_IMAGE_INSERTIONS = "imageInsertions"; + public static final String CTX_TEXT_INSERTIONS = "textInsertions"; + + public static void merge(InputStream in, + File outfile, + Map context, + Handlebars handlebars) throws Exception { + final File out = merge(in, context, handlebars); + if (empty(out)) die("merge: no outfiles generated"); + if (!out.renameTo(outfile)) die("merge: error renaming "+abs(out)+"->"+abs(outfile)); + } + + public static File merge(InputStream in, + Map context, + Handlebars handlebars) throws Exception { + return merge(in, context, handlebars, new ArrayList<>()); + } + + @SuppressWarnings("Duplicates") + public static File merge(InputStream in, + Map context, + Handlebars handlebars, + List validationErrors) throws Exception { + + final Map fieldMappings = (Map) context.get("fields"); + + // load the document + @Cleanup final PDDocument pdfDocument = PDDocument.load(in); + + // get the document catalog + final PDAcroForm acroForm = pdfDocument.getDocumentCatalog().getAcroForm(); + + // as there might not be an AcroForm entry a null check is necessary + if (acroForm != null) { + acroForm.setNeedAppearances(false); + + // Retrieve an individual field and set its value. + for (PDField field : acroForm.getFields()) { + try { + String fieldValue = fieldMappings == null ? null : fieldMappings.get(field.getFullyQualifiedName()); + if (!empty(fieldValue)) { + fieldValue = safeApply(context, handlebars, fieldValue, validationErrors); + if (fieldValue == null) continue; + } + if (field instanceof PDCheckBox) { + PDCheckBox box = (PDCheckBox) field; + if (!empty(fieldValue)) { + if (Boolean.valueOf(fieldValue)) { + box.check(); + } else { + box.unCheck(); + } + } + + } else { + String formValue = field.getValueAsString(); + if (formValue.equals(NULL_FORM_VALUE)) formValue = null; + if (empty(formValue) && field instanceof PDTextField) { + formValue = ((PDTextField) field).getDefaultValue(); + if (formValue.equals(NULL_FORM_VALUE)) formValue = null; + } + if (empty(formValue)) formValue = fieldValue; + if (!empty(formValue)) { + formValue = safeApply(context, handlebars, formValue, validationErrors); + if (formValue == null) continue; + try { + field.setValue(formValue); + } catch (Exception e) { + errorHandler.get().handleError("merge (field="+field+", value="+formValue+"): "+e, e); + } + } + } + } catch (Exception e) { + errorHandler.get().handleError("merge: "+e, e); + } + field.setReadOnly(true); + field.getCOSObject().setInt("Ff", 1); + } + // acroForm.flatten(); + acroForm.setNeedAppearances(false); + } + + // add images + final Map imageInsertions = (Map) context.get(CTX_IMAGE_INSERTIONS); + if (!empty(imageInsertions)) { + for (Object insertion : imageInsertions.values()) { + insertImage(pdfDocument, insertion, Base64ImageInsertion.class); + } + } + + // add text + final Map textInsertions = (Map) context.get(CTX_TEXT_INSERTIONS); + if (!empty(textInsertions)) { + for (Object insertion : textInsertions.values()) { + insertImage(pdfDocument, insertion, TextImageInsertion.class); + } + } + + final File output = temp(".pdf"); + + // Save and close the filled out form. + pdfDocument.getDocumentCatalog().setPageMode(PageMode.USE_THUMBS); + pdfDocument.save(output); + + if (validationErrors != null && !validationErrors.isEmpty()) { + errorHandler.get().handleError(validationErrors); + return null; + } + return output; + } + + public static String safeApply(Map context, Handlebars handlebars, String fieldValue, List validationErrors) { + try { + return HandlebarsUtil.apply(handlebars, fieldValue, context); + } catch (Exception e) { + if (validationErrors != null) { + log.warn("safeApply("+fieldValue+"): "+e); + validationErrors.add(fieldValue+"\t"+e.getMessage()); + return null; + } else { + throw e; + } + } + } + + protected static void insertImage(PDDocument pdfDocument, Object insert, Class clazz) throws IOException { + final ImageInsertion insertion; + if (insert instanceof ImageInsertion) { + insertion = (ImageInsertion) insert; + } else if (insert instanceof Map) { + insertion = instantiate(clazz); + insertion.init((Map) insert); + } else { + die("insertImage("+clazz.getSimpleName()+"): invalid object: "+insert); + return; + } + + // write image to temp file + File imageTemp = null; + try { + imageTemp = insertion.getImageFile(); + if (imageTemp != null) { + // create PD image + final PDImageXObject image = PDImageXObject.createFromFile(abs(imageTemp), pdfDocument); + final PDPageTree pages = pdfDocument.getDocumentCatalog().getPages(); + final float insertionHeight = insertion.getHeight(); + if (insertion.isWatermark()) { + for (PDPage page : pages) { + // set x, y, width and height to center insertion and maximize size on page + final float y = (page.getBBox().getHeight()/2.0f) - insertionHeight; + insertion.setX(20) + .setY(y) + .setWidth(page.getBBox().getWidth()-20) + .setHeight(page.getBBox().getHeight()-10); + insertImageOnPage(image, insertion, pdfDocument, page); + } + } else { + insertImageOnPage(image, insertion, pdfDocument, pages.get(insertion.getPage())); + } + } + } finally { + if (imageTemp != null && !imageTemp.delete()) log.warn("insertImage("+clazz.getSimpleName()+"): error deleting image file: "+abs(imageTemp)); + } + } + + private static void insertImageOnPage(PDImageXObject image, ImageInsertion insertion, PDDocument pdfDocument, PDPage page) throws IOException { + // open stream for writing inserted image + final PDPageContentStream contentStream = new PDPageContentStream(pdfDocument, page, PDPageContentStream.AppendMode.APPEND, true); + + // draw image on page + contentStream.drawImage(image, insertion.getX(), insertion.getY(), insertion.getWidth(), insertion.getHeight()); + contentStream.close(); + } + + public static void concatenate(List infiles, OutputStream out, long maxMemory, long maxDisk) throws IOException { + final PDFMergerUtility merger = new PDFMergerUtility(); + for (Object infile : infiles) { + if (infile instanceof File) { + merger.addSource((File) infile); + } else if (infile instanceof InputStream) { + merger.addSource((InputStream) infile); + } else if (infile instanceof String) { + merger.addSource((String) infile); + } else { + die("concatenate: invalid infile ("+infile.getClass().getName()+"): "+infile); + } + } + merger.setDestinationStream(out); + merger.mergeDocuments(MemoryUsageSetting.setupMixed(maxMemory, maxDisk)); + } + + public static void scrubAcroForm(File file, OutputStream output) throws IOException { + @Cleanup final InputStream pdfIn = FileUtils.openInputStream(file); + @Cleanup final PDDocument pdfDoc = PDDocument.load(pdfIn); + final PDAcroForm acroForm = pdfDoc.getDocumentCatalog().getAcroForm(); + + if (acroForm == null) { + Files.copy(file.toPath(), output); + } else { + acroForm.setNeedAppearances(false); + + File tempFile = temp(".pdf"); + pdfDoc.save(tempFile); + pdfDoc.close(); + Files.copy(tempFile.toPath(), output); + tempFile.delete(); + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/handlebars/RequiredVariableUndefinedException.java b/src/main/java/org/cobbzilla/util/handlebars/RequiredVariableUndefinedException.java new file mode 100644 index 0000000..5ae8715 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/handlebars/RequiredVariableUndefinedException.java @@ -0,0 +1,5 @@ +package org.cobbzilla.util.handlebars; + +public class RequiredVariableUndefinedException extends RuntimeException { + public RequiredVariableUndefinedException(String s) { super(s); } +} diff --git a/src/main/java/org/cobbzilla/util/handlebars/SimpleJurisdictionResolver.java b/src/main/java/org/cobbzilla/util/handlebars/SimpleJurisdictionResolver.java new file mode 100644 index 0000000..afd70e9 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/handlebars/SimpleJurisdictionResolver.java @@ -0,0 +1,22 @@ +package org.cobbzilla.util.handlebars; + +import org.cobbzilla.util.string.StringUtil; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +public class SimpleJurisdictionResolver implements JurisdictionResolver { + + public static final SimpleJurisdictionResolver instance = new SimpleJurisdictionResolver(); + + @Override public String usState(String value) { + return empty(value) || value.length() != 2 ? die("usState: invalid: " + value) : value.toUpperCase(); + } + + @Override public String usZip(String value) { + return empty(value) || value.length() != 5 || StringUtil.onlyDigits(value).length() != value.length() + ? die("usZip: invalid: " + value) + : value.toUpperCase(); + } + +} diff --git a/src/main/java/org/cobbzilla/util/handlebars/TextImageInsertion.java b/src/main/java/org/cobbzilla/util/handlebars/TextImageInsertion.java new file mode 100644 index 0000000..0fee7f0 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/handlebars/TextImageInsertion.java @@ -0,0 +1,170 @@ +package org.cobbzilla.util.handlebars; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.io.FileUtil.temp; + +@NoArgsConstructor @Accessors(chain=true) +public class TextImageInsertion extends ImageInsertion { + + public static final TextImageInsertion[] NO_TEXT_INSERTIONS = new TextImageInsertion[0]; + + @Getter @Setter private String content; + public void capitalizeContent() { content = content == null ? null : content.toUpperCase(); } + + @Getter @Setter private String fontFamily = "Arial"; + @Getter @Setter private String fontStyle = "plain"; + @Getter @Setter private String fontColor = "000000"; + @Getter @Setter private int fontSize = 14; + @Getter @Setter private int alpha = 255; + @Getter @Setter private int maxWidth = -1; + @Getter @Setter private int widthPadding = 10; + @Getter @Setter private int lineSpacing = 4; + + public TextImageInsertion(TextImageInsertion other) { super(other); } + public TextImageInsertion(String spec) { super(spec); } + + @Override protected void setField(String key, String value) { + switch (key) { + case "content": content = value; break; + case "fontFamily": fontFamily = value; break; + case "fontStyle": fontStyle = value; break; + case "fontColor": fontColor = value; break; + case "fontSize": fontSize = Integer.parseInt(value); break; + case "alpha": alpha = Integer.parseInt(value); break; + case "maxWidth": maxWidth = Integer.parseInt(value); break; + case "widthPadding": widthPadding = Integer.parseInt(value); break; + case "lineSpacing": lineSpacing = Integer.parseInt(value); break; + default: super.setField(key, value); + } + } + + @JsonIgnore private int getRed () { return (int) (Long.parseLong(fontColor, 16) & 0xff0000) >> 16; } + @JsonIgnore private int getGreen () { return (int) (Long.parseLong(fontColor, 16) & 0x00ff00) >> 8; } + @JsonIgnore private int getBlue () { return (int) (Long.parseLong(fontColor, 16) & 0x0000ff); } + + @JsonIgnore private Color getAwtFontColor() { return new Color(getRed(), getGreen(), getBlue(), getAlpha()); } + + @JsonIgnore private int getAwtFontStyle() { + switch (fontStyle.toLowerCase()) { + case "plain": return Font.PLAIN; + case "bold": return Font.BOLD; + case "italic": return Font.ITALIC; + default: return Font.PLAIN; + } + } + + // adapted from: https://stackoverflow.com/a/18800845/1251543 + @Override public File getImageFile() { + if (empty(getContent())) return null; + + Graphics2D g2d = getGraphics2D(); + + final ParsedText txt = getParsedText(g2d); + if (getWidth() == 0) setWidth(txt.width); + if (getHeight() == 0) setHeight(txt.height); + + g2d.dispose(); + + final BufferedImage img = new BufferedImage(txt.width, txt.height, BufferedImage.TYPE_INT_ARGB); + g2d = img.createGraphics(); + g2d.setRenderingHint(RenderingHints.KEY_ALPHA_INTERPOLATION, RenderingHints.VALUE_ALPHA_INTERPOLATION_QUALITY); + g2d.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON); + g2d.setRenderingHint(RenderingHints.KEY_COLOR_RENDERING, RenderingHints.VALUE_COLOR_RENDER_QUALITY); + g2d.setRenderingHint(RenderingHints.KEY_DITHERING, RenderingHints.VALUE_DITHER_ENABLE); + g2d.setRenderingHint(RenderingHints.KEY_FRACTIONALMETRICS, RenderingHints.VALUE_FRACTIONALMETRICS_ON); + g2d.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BICUBIC); + g2d.setRenderingHint(RenderingHints.KEY_RENDERING, RenderingHints.VALUE_RENDER_QUALITY); + g2d.setRenderingHint(RenderingHints.KEY_STROKE_CONTROL, RenderingHints.VALUE_STROKE_PURE); + g2d.setFont(getFont()); + + final FontMetrics fm = g2d.getFontMetrics(); + g2d.setColor(getAwtFontColor()); + for (int i=0; i widest) widest = txt.width; + } + + } else { + for (String inLine : inLines) { + final String[] words = inLine.split("\\s+"); + StringBuilder b = new StringBuilder(); + for (String word : words) { + int stringWidth = fm.stringWidth(b.toString() + " " + word); + if (stringWidth + getWidthPadding() > getMaxWidth()) { + if (b.length() == 0) die("getImageFile: word too long for maxWidth=" + maxWidth + ": " + word); + txt.lines.add(b.toString()); + b = new StringBuilder(word); + } else { + if (b.length() > 0) b.append(" "); + b.append(word); + if (stringWidth > widest) widest = stringWidth; + } + } + txt.lines.add(b.toString()); + } + } + txt.width = widest + getWidthPadding(); + txt.height = getLineY(fm, txt.lines.size()); + return txt; + } + + protected int getLineY(FontMetrics fm, int i) { return (i+1) * (fm.getAscent() + getLineSpacing()); } + + public int determineHeight() { return getParsedText().height; } + + private class ParsedText { + public java.util.List lines = new ArrayList<>(); + public int width; + public int height; + } +} diff --git a/src/main/java/org/cobbzilla/util/handlebars/WordDocxMerger.java b/src/main/java/org/cobbzilla/util/handlebars/WordDocxMerger.java new file mode 100644 index 0000000..17bf511 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/handlebars/WordDocxMerger.java @@ -0,0 +1,52 @@ +package org.cobbzilla.util.handlebars; + +import com.github.jknack.handlebars.Handlebars; +import lombok.extern.slf4j.Slf4j; +import org.apache.poi.xwpf.converter.xhtml.XHTMLConverter; +import org.apache.poi.xwpf.converter.xhtml.XHTMLOptions; +import org.apache.poi.xwpf.usermodel.XWPFDocument; +import org.cobbzilla.util.http.HtmlScreenCapture; +import org.cobbzilla.util.io.FileUtil; +import org.cobbzilla.util.xml.TidyHandlebarsSpanMerger; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Map; + +import static org.cobbzilla.util.io.FileUtil.temp; +import static org.cobbzilla.util.xml.TidyUtil.tidy; + +@Slf4j +public class WordDocxMerger { + + public static File merge(InputStream in, + Map context, + HtmlScreenCapture capture, + Handlebars handlebars) throws Exception { + + // convert to HTML + final XWPFDocument document = new XWPFDocument(in); + final File mergedHtml = temp(".html"); + try (OutputStream out = new FileOutputStream(mergedHtml)) { + final XHTMLOptions options = XHTMLOptions.create().setIgnoreStylesIfUnused(true); + XHTMLConverter.getInstance().convert(document, out, options); + } + + // - tidy HTML file + // - merge consecutive tags (which might occur in the middle of a {{variable}}) + // - replace HTML-entities encoded within handlebars templates (for example, convert ‘ and ’ to single-quote char) + // - apply Handlebars + String tidyHtml = tidy(mergedHtml, TidyHandlebarsSpanMerger.instance); + tidyHtml = TidyHandlebarsSpanMerger.scrubHandlebars(tidyHtml); + FileUtil.toFile(mergedHtml, HandlebarsUtil.apply(handlebars, tidyHtml, context)); + + // convert HTML -> PDF + final File pdfOutput = temp(".pdf"); + capture.capture(mergedHtml, pdfOutput); + + return pdfOutput; + } + +} diff --git a/src/main/java/org/cobbzilla/util/handlebars/main/PdfConcatMain.java b/src/main/java/org/cobbzilla/util/handlebars/main/PdfConcatMain.java new file mode 100644 index 0000000..2cc971b --- /dev/null +++ b/src/main/java/org/cobbzilla/util/handlebars/main/PdfConcatMain.java @@ -0,0 +1,20 @@ +package org.cobbzilla.util.handlebars.main; + +import lombok.Cleanup; +import org.cobbzilla.util.handlebars.PdfMerger; +import org.cobbzilla.util.main.BaseMain; + +import java.io.OutputStream; + +public class PdfConcatMain extends BaseMain { + + public static void main (String[] args) { main(PdfConcatMain.class, args); } + + @Override protected void run() throws Exception { + final PdfConcatOptions options = getOptions(); + @Cleanup final OutputStream out = options.getOut(); + PdfMerger.concatenate(options.getInfiles(), out, options.getMaxMemory(), options.getMaxDisk()); + out("success"); + } + +} diff --git a/src/main/java/org/cobbzilla/util/handlebars/main/PdfConcatOptions.java b/src/main/java/org/cobbzilla/util/handlebars/main/PdfConcatOptions.java new file mode 100644 index 0000000..b18552b --- /dev/null +++ b/src/main/java/org/cobbzilla/util/handlebars/main/PdfConcatOptions.java @@ -0,0 +1,39 @@ +package org.cobbzilla.util.handlebars.main; + +import lombok.Getter; +import lombok.Setter; +import org.cobbzilla.util.main.BaseMainOptions; +import org.kohsuke.args4j.Argument; +import org.kohsuke.args4j.Option; + +import java.io.File; +import java.io.OutputStream; +import java.util.List; + +public class PdfConcatOptions extends BaseMainOptions { + + public static final String USAGE_OUTFILE = "Output file. Default is stdout."; + public static final String OPT_OUTFILE = "-o"; + public static final String LONGOPT_OUTFILE= "--output"; + @Option(name=OPT_OUTFILE, aliases=LONGOPT_OUTFILE, usage=USAGE_OUTFILE) + @Getter @Setter private File outfile; + + public OutputStream getOut () { return outStream(outfile); } + + public static final String USAGE_INFILES = "Show help for this command"; + @Argument(usage=USAGE_INFILES) + @Getter @Setter private List infiles; + + public static final String USAGE_MAX_MEMORY = "Max memory to use. Default is unlimited"; + public static final String OPT_MAX_MEMORY = "-m"; + public static final String LONGOPT_MAX_MEMORY= "--max-memory"; + @Option(name=OPT_MAX_MEMORY, aliases=LONGOPT_MAX_MEMORY, usage=USAGE_MAX_MEMORY) + @Getter @Setter private long maxMemory = -1; + + public static final String USAGE_MAX_DISK = "Max disk to use. Default is unlimited"; + public static final String OPT_MAX_DISK = "-d"; + public static final String LONGOPT_MAX_DISK= "--max-disk"; + @Option(name=OPT_MAX_DISK, aliases=LONGOPT_MAX_DISK, usage=USAGE_MAX_DISK) + @Getter @Setter private long maxDisk = -1; + +} diff --git a/src/main/java/org/cobbzilla/util/handlebars/main/PdfMergeMain.java b/src/main/java/org/cobbzilla/util/handlebars/main/PdfMergeMain.java new file mode 100644 index 0000000..0a43a55 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/handlebars/main/PdfMergeMain.java @@ -0,0 +1,53 @@ +package org.cobbzilla.util.handlebars.main; + +import com.github.jknack.handlebars.Handlebars; +import lombok.Cleanup; +import lombok.Getter; +import org.cobbzilla.util.error.GeneralErrorHandler; +import org.cobbzilla.util.handlebars.PdfMerger; +import org.cobbzilla.util.main.BaseMain; +import org.cobbzilla.util.string.StringUtil; + +import java.io.File; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.io.FileUtil.abs; + +public class PdfMergeMain extends BaseMain { + + public static void main (String[] args) { main(PdfMergeMain.class, args); } + + @Getter protected Handlebars handlebars; + + @Override protected void run() throws Exception { + final PdfMergeOptions options = getOptions(); + + final List errors = new ArrayList<>(); + PdfMerger.setErrorHandler(new GeneralErrorHandler() { + @Override public T handleError(String message) { errors.add(message); return null; } + @Override public T handleError(String message, Exception e) { return handleError(message+": "+e.getClass().getSimpleName()+": "+e.getMessage()); } + @Override public T handleError(List validationErrors) { errors.addAll(validationErrors); return null; } + }); + @Cleanup final InputStream in = options.getInputStream(); + try { + if (options.hasOutfile()) { + final File outfile = options.getOutfile(); + PdfMerger.merge(in, outfile, options.getContext(), getHandlebars()); + out(abs(outfile)); + + } else { + final File output = PdfMerger.merge(in, options.getContext(), getHandlebars()); + out(abs(output)); + } + } catch (Exception e) { + err("Unexpected exception merging PDF: "+e.getClass().getSimpleName()+": "+e.getMessage()); + } + if (!empty(errors)) { + err(errors.size()+" error"+(errors.size() > 1 ? "s" : "")+" found when merging PDF:\n"+ StringUtil.toString(errors, "\n")); + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/handlebars/main/PdfMergeOptions.java b/src/main/java/org/cobbzilla/util/handlebars/main/PdfMergeOptions.java new file mode 100644 index 0000000..f9cd64e --- /dev/null +++ b/src/main/java/org/cobbzilla/util/handlebars/main/PdfMergeOptions.java @@ -0,0 +1,42 @@ +package org.cobbzilla.util.handlebars.main; + +import lombok.Getter; +import lombok.Setter; +import org.cobbzilla.util.json.JsonUtil; +import org.cobbzilla.util.main.BaseMainOptions; +import org.kohsuke.args4j.Option; + +import java.io.File; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +public class PdfMergeOptions extends BaseMainOptions { + + public static final String USAGE_INFILE = "Input file. Default is stdin"; + public static final String OPT_INFILE = "-i"; + public static final String LONGOPT_INFILE= "--infile"; + @Option(name=OPT_INFILE, aliases=LONGOPT_INFILE, usage=USAGE_INFILE) + @Getter @Setter private File infile; + + public InputStream getInputStream() { return inStream(getInfile()); } + + public static final String USAGE_CTXFILE = "Context file, must be a JSON map of String->Object"; + public static final String OPT_CTXFILE = "-c"; + public static final String LONGOPT_CTXFILE= "--context"; + @Option(name=OPT_CTXFILE, aliases=LONGOPT_CTXFILE, usage=USAGE_CTXFILE) + @Getter @Setter private File contextFile; + + public Map getContext() throws Exception { + if (contextFile == null) return new HashMap<>(); + return JsonUtil.fromJson(contextFile, Map.class); + } + + public static final String USAGE_OUTFILE = "Output file. Default is a random temp file"; + public static final String OPT_OUTFILE = "-o"; + public static final String LONGOPT_OUTFILE= "--outfile"; + @Option(name=OPT_OUTFILE, aliases=LONGOPT_OUTFILE, usage=USAGE_OUTFILE) + @Getter @Setter private File outfile; + public boolean hasOutfile () { return outfile != null; } + +} diff --git a/src/main/java/org/cobbzilla/util/http/ApiConnectionInfo.java b/src/main/java/org/cobbzilla/util/http/ApiConnectionInfo.java new file mode 100644 index 0000000..1e297b9 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/ApiConnectionInfo.java @@ -0,0 +1,27 @@ +package org.cobbzilla.util.http; + +import lombok.*; + +import static org.cobbzilla.util.reflect.ReflectionUtil.copy; + +@NoArgsConstructor @AllArgsConstructor @ToString(of={"baseUri", "user"}) +@EqualsAndHashCode(of={"baseUri", "user", "password"}) +public class ApiConnectionInfo { + + @Getter @Setter private String baseUri; + public boolean hasBaseUri () { return baseUri != null; } + + @Getter @Setter private String user; + public boolean hasUser () { return user != null; } + + @Getter @Setter private String password; + + public ApiConnectionInfo (String baseUri) { this.baseUri = baseUri; } + + public ApiConnectionInfo (ApiConnectionInfo other) { copy(this, other); } + + // alias for when this is used in json with snake_case naming conventions + public String getBase_uri () { return getBaseUri(); } + public void setBase_uri (String uri) { setBaseUri(uri); } + +} diff --git a/src/main/java/org/cobbzilla/util/http/CookieJar.java b/src/main/java/org/cobbzilla/util/http/CookieJar.java new file mode 100644 index 0000000..7d23ec4 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/CookieJar.java @@ -0,0 +1,57 @@ +package org.cobbzilla.util.http; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.NoArgsConstructor; +import org.apache.http.client.CookieStore; +import org.apache.http.cookie.Cookie; +import org.cobbzilla.util.collection.CaseInsensitiveStringKeyMap; + +import java.util.*; + +@NoArgsConstructor +public class CookieJar extends CaseInsensitiveStringKeyMap implements CookieStore { + + public CookieJar(List cookies) { for (HttpCookieBean cookie : cookies) add(cookie); } + + public CookieJar(HttpCookieBean cookie) { add(cookie); } + + public void add (HttpCookieBean cookie) { + if (cookie.expired()) { + remove(cookie.getName()); + } else { + put(cookie.getName(), cookie); + } + } + + @JsonIgnore + public String getRequestValue() { + final StringBuilder sb = new StringBuilder(); + for (String name : keySet()) { + if (sb.length() > 0) sb.append("; "); + sb.append(name).append("=").append(get(name).getValue()); + } + return sb.toString(); + } + + public List getCookiesList () { return new ArrayList<>(values()); } + + @Override public void addCookie(Cookie cookie) { add(new HttpCookieBean(cookie)); } + + @Override public List getCookies() { + final List cookies = new ArrayList<>(size()); + for (HttpCookieBean cookie : values()) { + cookies.add(cookie.toHttpClientCookie()); + } + return cookies; + } + + @Override public boolean clearExpired(Date date) { + final long expiration = date.getTime(); + final Set toRemove = new HashSet<>(); + for (HttpCookieBean cookie : values()) { + if (cookie.expired(expiration)) toRemove.add(cookie.getName()); + } + for (String name : toRemove) remove(name); + return false; + } +} \ No newline at end of file diff --git a/src/main/java/org/cobbzilla/util/http/HtmlScreenCapture.java b/src/main/java/org/cobbzilla/util/http/HtmlScreenCapture.java new file mode 100644 index 0000000..aa80826 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/HtmlScreenCapture.java @@ -0,0 +1,49 @@ +package org.cobbzilla.util.http; + +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; + +import java.io.File; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.now; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.io.StreamUtil.loadResourceAsStringOrDie; +import static org.cobbzilla.util.string.StringUtil.getPackagePath; +import static org.cobbzilla.util.system.Sleep.sleep; +import static org.cobbzilla.util.time.TimeUtil.formatDuration; + +@Slf4j +public class HtmlScreenCapture extends PhantomUtil { + + private static final long TIMEOUT = SECONDS.toMillis(60); + + public static final String SCRIPT = loadResourceAsStringOrDie(getPackagePath(HtmlScreenCapture.class)+"/html_screen_capture.js"); + + public synchronized void capture (String url, File file) { capture(url, file, TIMEOUT); } + + public synchronized void capture (String url, File file, long timeout) { + final String script = SCRIPT.replace("@@URL@@", url).replace("@@FILE@@", abs(file)); + try { + @Cleanup final PhantomJSHandle handle = execJs(script); + long start = now(); + while (file.length() == 0 && now() - start < timeout) sleep(200); + if (file.length() == 0 && now() - start >= timeout) { + sleep(5000); + if (file.length() == 0) die("capture: after " + formatDuration(timeout) + " file was never written to: " + abs(file)+", handle="+handle); + } + } catch (Exception e) { + die("capture: unexpected exception: "+e, e); + } + } + + public void capture (File in, File out) { + try { + capture(in.toURI().toString(), out); + } catch (Exception e) { + die("capture("+abs(in)+"): "+e, e); + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/http/HttpAuthType.java b/src/main/java/org/cobbzilla/util/http/HttpAuthType.java new file mode 100644 index 0000000..7e6c2b6 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/HttpAuthType.java @@ -0,0 +1,24 @@ +package org.cobbzilla.util.http; + +import com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.http.auth.AuthScheme; +import org.apache.http.impl.auth.BasicScheme; +import org.apache.http.impl.auth.DigestScheme; +import org.apache.http.impl.auth.KerberosScheme; + +import static org.cobbzilla.util.reflect.ReflectionUtil.instantiate; + +public enum HttpAuthType { + + basic (BasicScheme.class), + digest (DigestScheme.class), + kerberos (KerberosScheme.class); + + private final Class scheme; + HttpAuthType(Class scheme) { this.scheme = scheme; } + + public AuthScheme newScheme () { return instantiate(scheme); } + + @JsonCreator public static HttpAuthType create(String value) { return valueOf(value.toLowerCase()); } + +} diff --git a/src/main/java/org/cobbzilla/util/http/HttpCallStatus.java b/src/main/java/org/cobbzilla/util/http/HttpCallStatus.java new file mode 100644 index 0000000..1b7b500 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/HttpCallStatus.java @@ -0,0 +1,11 @@ +package org.cobbzilla.util.http; + +import com.fasterxml.jackson.annotation.JsonCreator; + +public enum HttpCallStatus { + + initialized, pending, requested, received_response, success, error, timeout; + + @JsonCreator public static HttpCallStatus fromString (String val) { return valueOf(val.toLowerCase()); } + +} diff --git a/src/main/java/org/cobbzilla/util/http/HttpClosingFilterInputStream.java b/src/main/java/org/cobbzilla/util/http/HttpClosingFilterInputStream.java new file mode 100644 index 0000000..615e5b3 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/HttpClosingFilterInputStream.java @@ -0,0 +1,30 @@ +package org.cobbzilla.util.http; + +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.impl.client.CloseableHttpClient; + +import java.io.FilterInputStream; +import java.io.IOException; + +public class HttpClosingFilterInputStream extends FilterInputStream { + + private final CloseableHttpClient httpClient; + + public HttpClosingFilterInputStream(CloseableHttpClient httpClient, + CloseableHttpResponse response) throws IOException { + super(response.getEntity().getContent()); + this.httpClient = httpClient; + } + + @Override public void close() throws IOException { + IOException ioe = null; + try { + super.close(); + } catch (IOException e) { + ioe = e; + } + httpClient.close(); + if (ioe != null) throw ioe; + } + +} diff --git a/src/main/java/org/cobbzilla/util/http/HttpContentTypes.java b/src/main/java/org/cobbzilla/util/http/HttpContentTypes.java new file mode 100644 index 0000000..2f06307 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/HttpContentTypes.java @@ -0,0 +1,125 @@ +package org.cobbzilla.util.http; + +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.collection.MapBuilder; +import org.cobbzilla.util.collection.NameAndValue; + +import java.util.Map; + +import static org.apache.commons.lang3.StringEscapeUtils.*; +import static org.apache.http.HttpHeaders.CONTENT_TYPE; +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +@Slf4j +public class HttpContentTypes { + + public static final String TEXT_HTML = "text/html"; + public static final String TEXT_PLAIN = "text/plain"; + public static final String TEXT_CSV = "text/csv"; + public static final String APPLICATION_JSON = "application/json"; + public static final String APPLICATION_XML = "application/xml"; + public static final String APPLICATION_PDF = "application/pdf"; + public static final String IMAGE_PNG = "image/png"; + public static final String IMAGE_JPEG = "image/jpg"; + public static final String IMAGE_GIF = "image/gif"; + public static final String APPLICATION_PEM_FILE = "application/x-pem-file"; + public static final String APPLICATION_PKCS12_FILE = "application/x-pkcs12"; + public static final String APPLICATION_CER_FILE = "application/x-x509-user-cert"; + public static final String APPLICATION_CRT_FILE = "application/x-x509-ca-cert"; + public static final String APPLICATION_OCTET_STREAM = "application/octet-stream"; + public static final String UNKNOWN = APPLICATION_OCTET_STREAM; + public static final String APPLICATION_ZIP = "application/zip"; + public static final String APPLICATION_JAR = "application/java-archive"; + public static final String APPLICATION_GZIP = "application/gzip"; + public static final String MULTIPART_FORM_DATA = "multipart/form-data"; + public static final String APPLICATION_FORM_URL_ENCODED = "application/x-www-form-urlencoded"; + // useful when constructing HttpRequestBeans that will be used against a JSON API + + private static NameAndValue[] nvHttp(String type) { return new NameAndValue[]{new NameAndValue(CONTENT_TYPE, type)}; } + + public static final NameAndValue[] NV_HTTP_JSON = nvHttp(APPLICATION_JSON); + public static final NameAndValue[] NV_HTTP_XML = nvHttp(APPLICATION_XML); + + public static final Map HTTP_CONTENT_TYPES = MapBuilder.build(new Object[][] { + { APPLICATION_JSON, NV_HTTP_JSON }, + { APPLICATION_XML, NV_HTTP_XML }, + }); + + public static final String CONTENT_TYPE_ANY = "*/*"; + + public static String contentType(String name) { + if (empty(name)) { + log.warn("contentType: no content-type could be determined for name (empty)"); + return APPLICATION_OCTET_STREAM; + } + final int dot = name.lastIndexOf('.'); + final String ext = (dot != -1 && dot != name.length()-1) ? name.substring(dot+1) : name; + switch (ext) { + case "htm": case "html": return TEXT_HTML; + case "png": return IMAGE_PNG; + case "jpg": case "jpeg": return IMAGE_JPEG; + case "gif": return IMAGE_GIF; + case "xml": return APPLICATION_XML; + case "pdf": return APPLICATION_PDF; + case "json": return APPLICATION_JSON; + case "gz": case "tgz": return APPLICATION_GZIP; + case "zip": return APPLICATION_ZIP; + case "jar": return APPLICATION_JAR; + case "txt": return TEXT_PLAIN; + case "csv": return TEXT_CSV; + case "pem": return APPLICATION_PEM_FILE; + case "p12": return APPLICATION_PKCS12_FILE; + case "cer": return APPLICATION_CER_FILE; + case "crt": return APPLICATION_CRT_FILE; + default: + log.warn("contentType: no content-type could be determined for name: "+name); + return APPLICATION_OCTET_STREAM; + } + } + + public static String fileExt (String contentType) { + switch (contentType) { + case TEXT_HTML: return ".html"; + case TEXT_PLAIN: return ".txt"; + case TEXT_CSV: return ".csv"; + case IMAGE_PNG: return ".png"; + case IMAGE_JPEG: return ".jpeg"; + case IMAGE_GIF: return ".gif"; + case APPLICATION_XML: return ".xml"; + case APPLICATION_PDF: return ".pdf"; + case APPLICATION_JSON: return ".json"; + case APPLICATION_ZIP: return ".zip"; + case APPLICATION_GZIP: return ".tar.gz"; + case APPLICATION_PEM_FILE: return ".pem"; + case APPLICATION_PKCS12_FILE: return ".p12"; + case APPLICATION_CER_FILE: return ".cer"; + case APPLICATION_CRT_FILE: return ".crt"; + default: return die("fileExt: no file extension could be determined for content-type: "+contentType); + } + } + + public static String fileExtNoDot (String contentType) { + return fileExt(contentType).substring(1); + } + + public static String escape(String mime, String data) { + switch (mime) { + case APPLICATION_XML: return escapeXml10(data); + case TEXT_HTML: return escapeHtml4(data); + } + return data; + } + + public static String unescape(String mime, String data) { + if (empty(data)) return data; + switch (mime) { + case APPLICATION_XML: return unescapeXml(data); + case TEXT_HTML: return unescapeHtml4(data); + } + return data; + } + + public static String multipartWithBoundary(String boundary) { return "multipart/form-data; boundary=" + boundary; } + +} diff --git a/src/main/java/org/cobbzilla/util/http/HttpCookieBean.java b/src/main/java/org/cobbzilla/util/http/HttpCookieBean.java new file mode 100644 index 0000000..32b7f26 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/HttpCookieBean.java @@ -0,0 +1,145 @@ +package org.cobbzilla.util.http; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.cookie.Cookie; +import org.apache.http.impl.cookie.BasicClientCookie; +import org.joda.time.DateTime; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import java.util.Date; +import java.util.StringTokenizer; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.daemon.ZillaRuntime.now; +import static org.cobbzilla.util.reflect.ReflectionUtil.copy; + +@NoArgsConstructor @Accessors(chain=true) @Slf4j +public class HttpCookieBean { + + public static final DateTimeFormatter[] EXPIRES_PATTERNS = { + DateTimeFormat.forPattern("E, dd MMM yyyy HH:mm:ss Z"), + DateTimeFormat.forPattern("E, dd-MMM-yyyy HH:mm:ss Z"), + DateTimeFormat.forPattern("E, dd MMM yyyy HH:mm:ss z"), + DateTimeFormat.forPattern("E, dd-MMM-yyyy HH:mm:ss z") + }; + + @Getter @Setter private String name; + @Getter @Setter private String value; + @Getter @Setter private String domain; + public boolean hasDomain () { return !empty(domain); } + + @Getter @Setter private String path; + @Getter @Setter private String expires; + @Getter @Setter private Long maxAge; + @Getter @Setter private boolean secure; + @Getter @Setter private boolean httpOnly; + + public HttpCookieBean(String name, String value) { this(name, value, null); } + + public HttpCookieBean(String name, String value, String domain) { + this.name = name; + this.value = value; + this.domain = domain; + } + + public HttpCookieBean (HttpCookieBean other) { copy(this, other); } + + public HttpCookieBean(Cookie cookie) { + this(cookie.getName(), cookie.getValue(), cookie.getDomain()); + path = cookie.getPath(); + secure = cookie.isSecure(); + final Date expiryDate = cookie.getExpiryDate(); + if (expiryDate != null) { + expires = EXPIRES_PATTERNS[0].print(expiryDate.getTime()); + } + } + + public static HttpCookieBean parse (String setCookie) { + final HttpCookieBean cookie = new HttpCookieBean(); + final StringTokenizer st = new StringTokenizer(setCookie, ";"); + while (st.hasMoreTokens()) { + final String token = st.nextToken().trim(); + if (cookie.name == null) { + // first element is the name=value + final String[] parts = token.split("="); + cookie.name = parts[0]; + cookie.value = parts.length == 1 ? "" : parts[1]; + + } else if (token.contains("=")) { + final String[] parts = token.split("="); + switch (parts[0].toLowerCase()) { + case "path": cookie.path = parts[1]; break; + case "domain": cookie.domain = parts[1]; break; + case "expires": cookie.expires = parts[1]; break; + case "max-age": cookie.maxAge = Long.valueOf(parts[1]); break; + default: log.warn("Unrecognized cookie attribute: "+parts[0]); + } + } else { + switch (token.toLowerCase()) { + case "httponly": cookie.httpOnly = true; break; + case "secure": cookie.secure = true; break; + default: log.warn("Unrecognized cookie attribute: "+token); + } + } + } + return cookie; + } + + public String toHeaderValue () { + StringBuilder sb = new StringBuilder(); + sb.append(name).append("=").append(value); + if (!empty(expires)) sb.append("; Expires=").append(expires); + if (maxAge != null) sb.append("; Max-Age=").append(maxAge); + if (!empty(path)) sb.append("; Path=").append(path); + if (!empty(domain)) sb.append("; Domain=").append(domain); + if (httpOnly) sb.append("; HttpOnly"); + if (secure) sb.append("; Secure"); + return sb.toString(); + } + + public String toRequestHeader () { return name + "=" + value; } + + public boolean expired () { + return (maxAge != null && maxAge <= 0) + || (expires != null && getExpiredDateTime().isBeforeNow()); + } + + public boolean expired (long expiration) { + return (maxAge != null && now() + maxAge < expiration) + || (expires != null && getExpiredDateTime().isBefore(expiration)); + } + + @JsonIgnore public Date getExpiryDate () { + if (maxAge != null) return new Date(now() + maxAge); + if (expires != null) return getExpiredDateTime().toDate(); + return null; + } + + protected DateTime getExpiredDateTime() { + if (empty(expires)) { + return null; + } + for (DateTimeFormatter formatter : EXPIRES_PATTERNS) { + try { + return formatter.parseDateTime(expires); + } catch (Exception ignored) {} + } + return die("getExpiredDateTime: unparseable 'expires' value for cookie "+name+": '"+expires+"'"); + } + + public Cookie toHttpClientCookie() { + final BasicClientCookie cookie = new BasicClientCookie(name, value); + cookie.setExpiryDate(getExpiryDate()); + cookie.setPath(path); + cookie.setDomain(domain); + cookie.setSecure(secure); + return cookie; + } +} diff --git a/src/main/java/org/cobbzilla/util/http/HttpMeta.java b/src/main/java/org/cobbzilla/util/http/HttpMeta.java new file mode 100644 index 0000000..e29a1ab --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/HttpMeta.java @@ -0,0 +1,42 @@ +package org.cobbzilla.util.http; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.io.FileUtil; + +import java.io.File; + +import static org.cobbzilla.util.io.FileUtil.abs; + +@NoArgsConstructor @Accessors(chain=true) @Slf4j +public class HttpMeta { + + public HttpMeta (String url) { this.url = url; } + + @Getter @Setter private String url; + @Getter @Setter private Long lastModified; + public boolean hasLastModified () { return lastModified != null; } + + @Getter @Setter private String etag; + public boolean hasEtag () { return etag != null; } + + public boolean shouldRefresh(File file) { + if (file == null) return true; + if (hasLastModified()) return getLastModified() > file.lastModified(); + if (hasEtag()) { + final File etagFile = new File(abs(file)+".etag"); + if (etagFile.exists()) { + try { + return !FileUtil.toString(etagFile).equals(etag); + } catch (Exception e) { + log.warn("shouldRefresh: "+e); + return true; + } + } + } + return true; + } +} diff --git a/src/main/java/org/cobbzilla/util/http/HttpMethods.java b/src/main/java/org/cobbzilla/util/http/HttpMethods.java new file mode 100644 index 0000000..dddedb2 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/HttpMethods.java @@ -0,0 +1,30 @@ +package org.cobbzilla.util.http; + +import org.apache.http.client.methods.*; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; + +public class HttpMethods { + + public static final String HEAD = "HEAD"; + public static final String GET = "GET"; + public static final String POST = "POST"; + public static final String PUT = "PUT"; + public static final String DELETE = "DELETE"; + public static final String PATCH = "PATCH"; + public static final String OPTIONS = "OPTIONS"; + + public static HttpRequestBase request(String method, String uri) { + switch (method) { + case HttpMethods.HEAD: return new HttpHead(uri); + case HttpMethods.GET: return new HttpGet(uri); + case HttpMethods.POST: return new HttpPost(uri); + case HttpMethods.PUT: return new HttpPut(uri); + case HttpMethods.DELETE: return new HttpDelete(uri); + case HttpMethods.PATCH: return new HttpPatch(uri); + case HttpMethods.OPTIONS:return new HttpOptions(uri); + } + return die("request: invalid HTTP method: "+method); + } + +} diff --git a/src/main/java/org/cobbzilla/util/http/HttpRequestBean.java b/src/main/java/org/cobbzilla/util/http/HttpRequestBean.java new file mode 100644 index 0000000..1659257 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/HttpRequestBean.java @@ -0,0 +1,186 @@ +package org.cobbzilla.util.http; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.*; +import lombok.experimental.Accessors; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScheme; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.AuthCache; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.entity.ContentType; +import org.apache.http.impl.client.BasicAuthCache; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.HttpClientBuilder; +import org.cobbzilla.util.collection.NameAndValue; +import org.cobbzilla.util.string.StringUtil; + +import java.io.InputStream; +import java.net.URI; +import java.util.*; + +import static org.apache.http.HttpHeaders.CONTENT_LENGTH; +import static org.apache.http.HttpHeaders.CONTENT_TYPE; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.http.HttpContentTypes.NV_HTTP_JSON; +import static org.cobbzilla.util.http.HttpMethods.*; +import static org.cobbzilla.util.reflect.ReflectionUtil.copy; +import static org.cobbzilla.util.system.CommandShell.execScript; + +/** + * A simple bean class that encapsulates the four things needed to make an HTTP request: + * * an HTTP request `method`, like GET, POST, PUT, etc. The default is GET + * * a `uri`, this is the only required parameter + * * an optional `entity`, representing the request body to send for methods like POST and PUT + * * an optional array of `headers`, name/value pairs (allowing duplicates) that will be the HTTP request headers + */ +@NoArgsConstructor @ToString(of={"method", "uri"}) @Accessors(chain=true) +public class HttpRequestBean { + + @Getter @Setter private String method = GET; + @Getter @Setter private String uri; + + @Getter @Setter private String entity; + @Getter @Setter private InputStream entityInputStream; + @Getter @Setter private Boolean discardResponseEntity; + public boolean discardResponseEntity () { return discardResponseEntity != null && discardResponseEntity; } + + public HttpRequestBean(HttpRequestBean request) { copy(this, request); } + + public boolean hasData () { return entity != null; } + public boolean hasStream () { return entityInputStream != null; } + + @Getter @Setter private List headers = new ArrayList<>(); + public HttpRequestBean withHeader (String name, String value) { setHeader(name, value); return this; } + public HttpRequestBean setHeader (String name, String value) { + headers.add(new NameAndValue(name, value)); + return this; + } + public boolean hasHeaders () { return !empty(headers); } + public boolean hasHeader(String name) { return NameAndValue.find(getHeaders(), name) != null; } + + public HttpRequestBean (String uri) { this(GET, uri, null); } + + public HttpRequestBean (String method, String uri) { this(method, uri, null); } + + public HttpRequestBean (String method, String uri, String entity) { + this.method = method; + this.uri = uri; + this.entity = entity; + } + + public HttpRequestBean (String method, String uri, String entity, List headers) { + this(method, uri, entity); + this.headers = headers; + } + + public HttpRequestBean (String method, String uri, String entity, NameAndValue[] headers) { + this(method, uri, entity); + this.headers = Arrays.asList(headers); + } + + public HttpRequestBean (String method, String uri, InputStream entity, String name, NameAndValue[] headers) { + this(method, uri); + this.entity = name; + this.entityInputStream = entity; + this.headers = new ArrayList(Arrays.asList(headers)); + } + + public Map toMap () { + final Map map = new LinkedHashMap<>(); + map.put("method", method); + map.put("uri", uri); + if (!empty(headers)) map.put("headers", headers.toArray()); + map.put("entity", hasContentType() ? HttpContentTypes.escape(getContentType().getMimeType(), entity) : entity); + return map; + } + + @Getter(lazy=true, value=AccessLevel.PRIVATE) private final URI _uri = initURI(); + + private URI initURI() { return StringUtil.uriOrDie(uri); } + + @JsonIgnore public String getHost () { return get_uri().getHost(); } + @JsonIgnore public int getPort () { return get_uri().getPort(); } + @JsonIgnore public String getPath () { return get_uri().getPath(); } + + @JsonIgnore @Getter(lazy=true) private final HttpHost httpHost = initHttpHost(); + private HttpHost initHttpHost() { return new HttpHost(getHost(), getPort(), get_uri().getScheme()); } + + @Getter @Setter private HttpAuthType authType; + @Getter @Setter private String authUsername; + @Getter @Setter private String authPassword; + + public boolean hasAuth () { return authType != null; } + + public HttpRequestBean setAuth(HttpAuthType authType, String name, String password) { + setAuthType(authType); + setAuthUsername(name); + setAuthPassword(password); + return this; + } + + @JsonIgnore public ContentType getContentType() { + if (!hasHeaders()) return null; + final String value = getFirstHeaderValue(CONTENT_TYPE); + if (empty(value)) return null; + return ContentType.parse(value); + } + public boolean hasContentType () { return getContentType() != null; } + + @JsonIgnore public Long getContentLength() { + if (!hasHeaders()) return null; + final String value = getFirstHeaderValue(CONTENT_LENGTH); + if (empty(value)) return null; + return Long.parseLong(value); + } + public boolean hasContentLength () { return getContentLength() != null; } + + private String getFirstHeaderValue(String name) { + if (!hasHeaders()) return null; + for (NameAndValue header : getHeaders()) if (header.getName().equalsIgnoreCase(name)) return header.getValue(); + return null; + } + + public static HttpRequestBean get (String path) { return new HttpRequestBean(GET, path); } + public static HttpRequestBean put (String path, String json) { return new HttpRequestBean(PUT, path, json); } + public static HttpRequestBean post (String path, String json) { return new HttpRequestBean(POST, path, json); } + public static HttpRequestBean delete(String path) { return new HttpRequestBean(DELETE, path); } + + public static HttpRequestBean putJson (String path, String json) { return new HttpRequestBean(PUT, path, json, NV_HTTP_JSON); } + public static HttpRequestBean postJson(String path, String json) { return new HttpRequestBean(POST, path, json, NV_HTTP_JSON); } + + public String cURL () { + final StringBuilder b = new StringBuilder("curl '"+getUri()).append("'"); + for (NameAndValue header : getHeaders()) { + final String name = header.getName(); + b.append(" -H '").append(name).append(": ").append(header.getValue()).append("'"); + } + if (getMethod().equals(PUT) || getMethod().equals(POST)) { + b.append(" --data-binary '").append(getEntity()).append("'"); + } + return b.toString(); + } + + + public HttpResponseBean curl() { + return new HttpResponseBean().setStatus(200).setEntityBytes(execScript(cURL()).getBytes()); + } + + public HttpClientBuilder initClientBuilder(HttpClientBuilder clientBuilder) { + if (!hasAuth()) return clientBuilder; + final HttpClientContext localContext = HttpClientContext.create(); + final BasicCredentialsProvider credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials( + new AuthScope(getHost(), getPort()), + new UsernamePasswordCredentials(getAuthUsername(), getAuthPassword())); + + final AuthCache authCache = new BasicAuthCache(); + final AuthScheme authScheme = getAuthType().newScheme(); + authCache.put(getHttpHost(), authScheme); + + localContext.setAuthCache(authCache); + clientBuilder.setDefaultCredentialsProvider(credsProvider); + return clientBuilder; + } +} diff --git a/src/main/java/org/cobbzilla/util/http/HttpRequestPreprocessor.java b/src/main/java/org/cobbzilla/util/http/HttpRequestPreprocessor.java new file mode 100644 index 0000000..97da660 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/HttpRequestPreprocessor.java @@ -0,0 +1,7 @@ +package org.cobbzilla.util.http; + +public interface HttpRequestPreprocessor { + + HttpRequestBean preProcess (HttpRequestBean request); + +} diff --git a/src/main/java/org/cobbzilla/util/http/HttpResponseBean.java b/src/main/java/org/cobbzilla/util/http/HttpResponseBean.java new file mode 100644 index 0000000..65c734c --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/HttpResponseBean.java @@ -0,0 +1,125 @@ +package org.cobbzilla.util.http; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.apache.http.Header; +import org.apache.http.HttpHeaders; +import org.cobbzilla.util.collection.NameAndValue; +import org.cobbzilla.util.json.JsonUtil; + +import java.io.*; +import java.util.*; + +import static org.apache.http.HttpHeaders.CONTENT_LENGTH; +import static org.apache.http.HttpHeaders.CONTENT_TYPE; +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.io.FileUtil.mkdirOrDie; +import static org.cobbzilla.util.string.StringUtil.UTF8cs; + +@Slf4j @Accessors(chain=true) @ToString(of={"status", "headers"}) +public class HttpResponseBean { + + public static final HttpResponseBean OK = new HttpResponseBean().setStatus(HttpStatusCodes.OK); + + @Getter @Setter private int status; + @Getter @Setter private List headers; + @JsonIgnore @Getter private byte[] entity; + @Getter @Setter private long contentLength; + @Getter @Setter private String contentType; + + @JsonIgnore public boolean isOk() { return (status / 100) == 2; } + + public Map toMap () { + final Map map = new LinkedHashMap<>(); + map.put("status", status); + if (!empty(headers)) map.put("headers", headers.toArray()); + map.put("entity", hasContentType() ? HttpContentTypes.escape(contentType(), getEntityString()) : getEntityString()); + return map; + } + + public boolean hasHeader (String name) { return !empty(getHeaderValues(name)); } + public boolean hasContentType () { return contentType != null || hasHeader(HttpHeaders.CONTENT_TYPE); } + public String contentType () { return contentType != null ? contentType : getFirstHeaderValue(HttpHeaders.CONTENT_TYPE); } + + public void addHeader(String name, String value) { + if (headers == null) headers = new ArrayList<>(); + if (name.equalsIgnoreCase(CONTENT_TYPE)) setContentType(value); + else if (name.equalsIgnoreCase(CONTENT_LENGTH)) setContentLength(Long.valueOf(value)); + headers.add(new NameAndValue(name, value)); + } + + public HttpResponseBean setEntityBytes(byte[] bytes) { this.entity = bytes; return this; } + + public HttpResponseBean setEntity (InputStream entity) { + try { + this.entity = entity == null ? null : IOUtils.toByteArray(entity); + return this; + } catch (IOException e) { + return die("setEntity: error reading stream: " + e, e); + } + } + + public boolean hasEntity () { return !empty(entity); } + + public String getEntityString () { + try { + return entity == null ? null : new String(entity, UTF8cs); + } catch (Exception e) { + log.warn("getEntityString: error parsing bytes: "+e); + return null; + } + } + + public T getEntity (Class clazz) { + return entity == null ? null : JsonUtil.fromJsonOrDie(getEntityString(), clazz); + } + + public Collection getHeaderValues (String name) { + final List values = new ArrayList<>(); + if (!empty(headers)) for (NameAndValue header : headers) if (header.getName().equalsIgnoreCase(name)) values.add(header.getValue()); + return values; + } + + + public String getFirstHeaderValue (String name) { + if (empty(headers)) return null; + for (NameAndValue header : headers) if (header.getName().equalsIgnoreCase(name)) return header.getValue(); + return null; + } + + public HttpResponseBean setHttpHeaders(Header[] headers) { + for (Header header : headers) { + addHeader(header.getName(), header.getValue());; + } + return this; + } + + public HttpResponseBean setHttpHeaders(Map> h) { + if (empty(h)) return this; + for (Map.Entry> e : h.entrySet()) { + if (!empty(e.getKey())) { + for (String v : e.getValue()) { + if (!empty(v)) addHeader(e.getKey(), v); + } + } + } + return this; + } + + public File toFile(File file) { + if (!isOk()) return die("unexpected HTTP response: "+this); + if (!file.getParentFile().exists()) mkdirOrDie(file.getParentFile()); + try (OutputStream out = new FileOutputStream(file)) { + IOUtils.copyLarge(new ByteArrayInputStream(getEntity()), out); + return file; + } catch (Exception e) { + return die("toFile: "+e, e); + } + } +} diff --git a/src/main/java/org/cobbzilla/util/http/HttpResponseHandler.java b/src/main/java/org/cobbzilla/util/http/HttpResponseHandler.java new file mode 100644 index 0000000..303646e --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/HttpResponseHandler.java @@ -0,0 +1,9 @@ +package org.cobbzilla.util.http; + +public interface HttpResponseHandler { + + boolean isSuccess (HttpRequestBean request, HttpResponseBean response); + void success (HttpRequestBean request, HttpResponseBean response); + void failure (HttpRequestBean request, HttpResponseBean response); + +} diff --git a/src/main/java/org/cobbzilla/util/http/HttpResponsePostprocessor.java b/src/main/java/org/cobbzilla/util/http/HttpResponsePostprocessor.java new file mode 100644 index 0000000..29a3e46 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/HttpResponsePostprocessor.java @@ -0,0 +1,7 @@ +package org.cobbzilla.util.http; + +public interface HttpResponsePostprocessor { + + HttpResponseBean postProcess (HttpResponseBean response); + +} diff --git a/src/main/java/org/cobbzilla/util/http/HttpSchemes.java b/src/main/java/org/cobbzilla/util/http/HttpSchemes.java new file mode 100644 index 0000000..a6d87d4 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/HttpSchemes.java @@ -0,0 +1,18 @@ +package org.cobbzilla.util.http; + +public enum HttpSchemes { + + http, https; + + public static HttpSchemes from(String s) { + return valueOf(s.toLowerCase()); + } + + public static boolean isValid(String s) { + s = s.toLowerCase(); + for (HttpSchemes scheme : values()) { + if (s.startsWith(scheme.name())) return true; + } + return false; + } +} diff --git a/src/main/java/org/cobbzilla/util/http/HttpStatusCodes.java b/src/main/java/org/cobbzilla/util/http/HttpStatusCodes.java new file mode 100644 index 0000000..9d6c723 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/HttpStatusCodes.java @@ -0,0 +1,21 @@ +package org.cobbzilla.util.http; + +public class HttpStatusCodes { + + public static final int UNPROCESSABLE_ENTITY = 422; + public static final int OK = 200; + public static final int CREATED = 201; + public static final int ACCEPTED = 202; + public static final int NON_AUTHORITATIVE_INFO = 203; + public static final int NO_CONTENT = 204; + public static final int FOUND = 302; + public static final int UNAUTHORIZED = 401; + public static final int FORBIDDEN = 403; + public static final int NOT_FOUND = 404; + public static final int PRECONDITION_FAILED = 412; + public static final int UNSUPPORTED_MEDIA_TYPE = 415; + public static final int SERVER_ERROR = 500; + public static final int SERVER_UNAVAILABLE = 503; + public static final int GATEWAY_TIMEOUT = 504; + +} diff --git a/src/main/java/org/cobbzilla/util/http/HttpUtil.java b/src/main/java/org/cobbzilla/util/http/HttpUtil.java new file mode 100644 index 0000000..15235ec --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/HttpUtil.java @@ -0,0 +1,363 @@ +package org.cobbzilla.util.http; + +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.exec.CommandLine; +import org.apache.commons.io.IOUtils; +import org.apache.http.Header; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.*; +import org.apache.http.entity.InputStreamEntity; +import org.apache.http.entity.StringEntity; +import org.apache.http.entity.mime.MultipartEntityBuilder; +import org.apache.http.entity.mime.content.FileBody; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicHeader; +import org.apache.http.util.EntityUtils; +import org.cobbzilla.util.collection.NameAndValue; +import org.cobbzilla.util.string.StringUtil; +import org.cobbzilla.util.system.CommandResult; +import org.cobbzilla.util.system.CommandShell; +import org.cobbzilla.util.system.Sleep; + +import java.io.*; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLDecoder; +import java.nio.charset.Charset; +import java.util.LinkedHashMap; +import java.util.Map; + +import static com.google.common.net.HttpHeaders.CONTENT_DISPOSITION; +import static org.apache.http.HttpHeaders.*; +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.hexnow; +import static org.cobbzilla.util.http.HttpContentTypes.contentType; +import static org.cobbzilla.util.http.HttpMethods.*; +import static org.cobbzilla.util.http.HttpStatusCodes.NO_CONTENT; +import static org.cobbzilla.util.http.URIUtil.getFileExt; +import static org.cobbzilla.util.io.FileUtil.getDefaultTempDir; +import static org.cobbzilla.util.security.CryptStream.BUFFER_SIZE; +import static org.cobbzilla.util.string.StringUtil.CRLF; +import static org.cobbzilla.util.system.Sleep.sleep; +import static org.cobbzilla.util.time.TimeUtil.DATE_FORMAT_LAST_MODIFIED; + +@Slf4j +public class HttpUtil { + + public static final String DEFAULT_CERT_NAME = "ssl-https"; + + public static Map queryParams(URL url) throws UnsupportedEncodingException { + return queryParams(url, StringUtil.UTF8); + } + + // from: http://stackoverflow.com/a/13592567 + public static Map queryParams(URL url, String encoding) throws UnsupportedEncodingException { + final Map query_pairs = new LinkedHashMap<>(); + final String query = url.getQuery(); + final String[] pairs = query.split("&"); + for (String pair : pairs) { + final int idx = pair.indexOf("="); + query_pairs.put(URLDecoder.decode(pair.substring(0, idx), encoding), URLDecoder.decode(pair.substring(idx + 1), encoding)); + } + return query_pairs; + } + + public static InputStream get (String urlString) throws IOException { return get(urlString, null); } + + public static InputStream get (String urlString, Map headers) throws IOException { + final URL url = new URL(urlString); + final HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); + if (headers != null) { + for (Map.Entry h : headers.entrySet()) { + urlConnection.setRequestProperty(h.getKey(), h.getValue()); + } + } + return urlConnection.getInputStream(); + } + + public static HttpResponseBean upload (String url, + File file, + Map headers) throws IOException { + @Cleanup final CloseableHttpClient client = HttpClients.createDefault(); + final HttpPost method = new HttpPost(url); + final FileBody fileBody = new FileBody(file); + MultipartEntityBuilder builder = MultipartEntityBuilder.create().addPart("file", fileBody); + method.setEntity(builder.build()); + + if (headers != null) { + for (Map.Entry header : headers.entrySet()) { + method.addHeader(new BasicHeader(header.getKey(), header.getValue())); + } + } + + @Cleanup final CloseableHttpResponse response = client.execute(method); + + final HttpResponseBean responseBean = new HttpResponseBean() + .setEntityBytes(EntityUtils.toByteArray(response.getEntity())) + .setHttpHeaders(response.getAllHeaders()) + .setStatus(response.getStatusLine().getStatusCode()); + return responseBean; + } + + public static final int DEFAULT_RETRIES = 3; + + public static File url2file (String url) throws IOException { + return url2file(url, null, DEFAULT_RETRIES); + } + public static File url2file (String url, String file) throws IOException { + return url2file(url, file == null ? null : new File(file), DEFAULT_RETRIES); + } + public static File url2file (String url, File file) throws IOException { + return url2file(url, file, DEFAULT_RETRIES); + } + public static File url2file (String url, File file, int retries) throws IOException { + if (file == null) file = File.createTempFile("url2file-", getFileExt((url)), getDefaultTempDir()); + IOException lastException = null; + long sleep = 100; + for (int i=0; i BUFFER_SIZE)) { + connection.setChunkedStreamingMode(BUFFER_SIZE); + } + for (NameAndValue header : request.getHeaders()) { + connection.setRequestProperty(header.getName(), header.getValue()); + } + + @Cleanup final OutputStream output = connection.getOutputStream(); + @Cleanup final PrintWriter writer = new PrintWriter(new OutputStreamWriter(output, Charset.defaultCharset()), true); + writer.append("--").append(boundary).append(CRLF); + final String filename = request.getEntity(); + addStreamHeader(writer, CONTENT_DISPOSITION, "form-data; name=\"file\"; filename=\""+ filename +"\""); + addStreamHeader(writer, CONTENT_TYPE, contentType(filename)); + writer.append(CRLF).flush(); + IOUtils.copy(request.getEntityInputStream(), output); + output.flush(); + writer.append(CRLF); + writer.append("--").append(boundary).append("--").append(CRLF).flush(); + + final HttpResponseBean response = new HttpResponseBean() + .setStatus(connection.getResponseCode()) + .setHttpHeaders(connection.getHeaderFields()); + if (!request.discardResponseEntity()) { + try { + response.setEntity(connection.getInputStream()); + } catch (IOException ioe) { + response.setEntity(connection.getErrorStream()); + } + } + return response; + } catch (Exception e) { + return die("getStreamResponse: "+e, e); + } + } + + private static PrintWriter addStreamHeader(PrintWriter writer, String name, String value) { + writer.append(name).append(": ").append(value).append(CRLF); + return writer; + } + + public static HttpResponseBean getResponse(String urlString) throws IOException { + + final HttpResponseBean bean = new HttpResponseBean(); + @Cleanup final CloseableHttpClient client = HttpClients.createDefault(); + final HttpResponse response = client.execute(new HttpGet(urlString.trim())); + + for (Header header : response.getAllHeaders()) { + bean.addHeader(header.getName(), header.getValue()); + } + + bean.setStatus(response.getStatusLine().getStatusCode()); + if (response.getEntity() != null) { + final Header contentType = response.getEntity().getContentType(); + if (contentType != null) bean.setContentType(contentType.getValue()); + + bean.setContentLength(response.getEntity().getContentLength()); + bean.setEntity(response.getEntity().getContent()); + } + + return bean; + } + + public static HttpUriRequest initHttpRequest(HttpRequestBean requestBean) { + try { + final HttpUriRequest request; + switch (requestBean.getMethod()) { + case HEAD: + request = new HttpHead(requestBean.getUri()); + break; + + case GET: + request = new HttpGet(requestBean.getUri()); + break; + + case POST: + request = new HttpPost(requestBean.getUri()); + break; + + case PUT: + request = new HttpPut(requestBean.getUri()); + break; + + case PATCH: + request = new HttpPatch(requestBean.getUri()); + break; + + case DELETE: + request = new HttpDelete(requestBean.getUri()); + break; + + default: + return die("Invalid request method: " + requestBean.getMethod()); + } + + if (requestBean.hasData() && request instanceof HttpEntityEnclosingRequestBase) { + setData(requestBean.getEntity(), (HttpEntityEnclosingRequestBase) request); + } + + return request; + + } catch (UnsupportedEncodingException e) { + return die("initHttpRequest: " + e, e); + } + } + + private static void setData(Object data, HttpEntityEnclosingRequestBase request) throws UnsupportedEncodingException { + if (data == null) return; + if (data instanceof String) { + request.setEntity(new StringEntity((String) data)); + } else if (data instanceof InputStream) { + request.setEntity(new InputStreamEntity((InputStream) data)); + } else { + throw new IllegalArgumentException("Unsupported request entity type: "+data.getClass().getName()); + } + } + + public static String getContentType(HttpResponse response) { + final Header contentTypeHeader = response.getFirstHeader(CONTENT_TYPE); + return (contentTypeHeader == null) ? null : contentTypeHeader.getValue(); + } + + public static boolean isOk(String url) { return isOk(url, URIUtil.getHost(url)); } + + public static boolean isOk(String url, String host) { + final CommandLine command = new CommandLine("curl") + .addArgument("--insecure") // since we are requested via the IP address, the cert will not match + .addArgument("--header").addArgument("Host: " + host) // pass FQDN via Host header + .addArgument("--silent") + .addArgument("--location") // follow redirects + .addArgument("--write-out").addArgument("%{http_code}") // just print status code + .addArgument("--output").addArgument("/dev/null") // and ignore data + .addArgument(url); + try { + final CommandResult result = CommandShell.exec(command); + final String statusCode = result.getStdout(); + return result.isZeroExitStatus() && statusCode != null && statusCode.trim().startsWith("2"); + + } catch (IOException e) { + log.warn("isOk: Error fetching " + url + " with Host header=" + host + ": " + e); + return false; + } + } + + public static boolean isOk(String url, String host, int maxTries, long sleepUnit) { + long sleep = sleepUnit; + for (int i = 0; i < maxTries; i++) { + if (i > 0) { + Sleep.sleep(sleep); + sleep *= 2; + } + if (isOk(url, host)) return true; + } + return false; + } + + public static HttpMeta getHeadMetadata(HttpRequestBean request) throws IOException { + final HttpResponseBean headResponse = HttpUtil.getResponse(new HttpRequestBean(request).setMethod(HEAD)); + if (!headResponse.isOk()) return die("HTTP HEAD response was not 200: "+headResponse); + + final HttpMeta meta = new HttpMeta(request.getUri()); + + final String lastModString = headResponse.getFirstHeaderValue(LAST_MODIFIED); + if (lastModString != null) meta.setLastModified(DATE_FORMAT_LAST_MODIFIED.parseMillis(lastModString)); + + final String etag = headResponse.getFirstHeaderValue(ETAG); + if (etag != null) meta.setEtag(etag); + + return meta; + } +} diff --git a/src/main/java/org/cobbzilla/util/http/PhantomJSHandle.java b/src/main/java/org/cobbzilla/util/http/PhantomJSHandle.java new file mode 100644 index 0000000..49b90ca --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/PhantomJSHandle.java @@ -0,0 +1,24 @@ +package org.cobbzilla.util.http; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.openqa.selenium.phantomjs.PhantomJSDriver; +import org.openqa.selenium.remote.ErrorHandler; +import org.openqa.selenium.remote.Response; + +import java.io.Closeable; + +@AllArgsConstructor @Slf4j +public class PhantomJSHandle extends ErrorHandler implements Closeable { + + @Getter final PhantomJSDriver driver; + + @Override public void close() { + if (driver != null) { + driver.close(); + driver.quit(); + } + } + +} \ No newline at end of file diff --git a/src/main/java/org/cobbzilla/util/http/PhantomUtil.java b/src/main/java/org/cobbzilla/util/http/PhantomUtil.java new file mode 100644 index 0000000..f909f7a --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/PhantomUtil.java @@ -0,0 +1,47 @@ +package org.cobbzilla.util.http; + +import io.github.bonigarcia.wdm.WebDriverManager; +import lombok.extern.slf4j.Slf4j; +import org.openqa.selenium.phantomjs.PhantomJSDriver; +import org.openqa.selenium.remote.DesiredCapabilities; + +import java.io.File; + +import static org.cobbzilla.util.io.FileUtil.abs; + +@Slf4j +public class PhantomUtil { + + static { WebDriverManager.phantomjs().setup(); } + + // ensures static-initializer above gets run + public static void init () {} + + private PhantomJSDriver defaultDriver() { + final DesiredCapabilities capabilities = new DesiredCapabilities(); + capabilities.setJavascriptEnabled(true); + return new PhantomJSDriver(capabilities); + } + + public PhantomJSHandle execJs(String script) { + final PhantomJSDriver driver = defaultDriver(); + final PhantomJSHandle handle = new PhantomJSHandle(driver); + driver.setErrorHandler(handle); + driver.executePhantomJS(script); + return handle; + } + + public static final String LOAD_AND_EXEC = "var page = require('webpage').create();\n" + + "page.open('@@URL@@', function() {\n" + + " page.evaluateJavaScript('@@JS@@');\n" + + "});\n"; + + public void loadPage (File file) { loadPageAndExec(file, "console.log('successfully loaded "+abs(file)+"')"); } + public void loadPage(String url) { loadPageAndExec(url, "console.log('successfully loaded "+url+"')"); } + + public void loadPageAndExec(File file, String script) { loadPageAndExec("file://"+abs(file), script); } + + public void loadPageAndExec(String url, String script) { + execJs(LOAD_AND_EXEC.replace("@@URL@@", url).replace("@@JS@@", script)); + } +} diff --git a/src/main/java/org/cobbzilla/util/http/PooledHttpClientFactory.java b/src/main/java/org/cobbzilla/util/http/PooledHttpClientFactory.java new file mode 100644 index 0000000..50578ac --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/PooledHttpClientFactory.java @@ -0,0 +1,32 @@ +package org.cobbzilla.util.http; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.apache.http.HttpHost; +import org.apache.http.conn.routing.HttpRoute; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.cobbzilla.util.reflect.ObjectFactory; + +import java.util.Map; + +@AllArgsConstructor +public class PooledHttpClientFactory implements ObjectFactory { + + @Getter private String host; + @Getter private int maxConnections; + + @Override public CloseableHttpClient create() { + final PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); + cm.setMaxTotal(maxConnections); + cm.setMaxPerRoute(new HttpRoute(new HttpHost(host)), maxConnections); + return HttpClients.custom() + .setConnectionManager(cm) + .setConnectionManagerShared(true) + .build(); + } + + @Override public CloseableHttpClient create(Map ctx) { return create(); } + +} diff --git a/src/main/java/org/cobbzilla/util/http/URIBean.java b/src/main/java/org/cobbzilla/util/http/URIBean.java new file mode 100644 index 0000000..629e963 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/URIBean.java @@ -0,0 +1,38 @@ +package org.cobbzilla.util.http; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; +import lombok.Setter; +import org.apache.http.client.utils.URIBuilder; + +import java.net.URI; +import java.net.URISyntaxException; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +public class URIBean { + + @Getter @Setter private String scheme = "http"; + @Getter @Setter private String host; + @Getter @Setter private int port = 80; + @Getter @Setter private String path = "/"; + @Getter @Setter private String query; + + public URI toURI() { + try { + return new URIBuilder() + .setScheme(scheme) + .setHost(host) + .setPort(port) + .setPath(path) + .setCustomQuery(query) + .build(); + } catch (URISyntaxException e) { + return die("toURI: "+e, e); + } + } + + @JsonIgnore public String getFullPath() { return getPath() + (empty(query) ? "" : "?" + query); } + +} diff --git a/src/main/java/org/cobbzilla/util/http/URIUtil.java b/src/main/java/org/cobbzilla/util/http/URIUtil.java new file mode 100644 index 0000000..3834e50 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/URIUtil.java @@ -0,0 +1,71 @@ +package org.cobbzilla.util.http; + +import java.net.URI; +import java.net.URISyntaxException; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +public class URIUtil { + + public static URI toUri(String uri) { + try { return new URI(uri); } catch (URISyntaxException e) { + return die("Invalid URI: " + uri); + } + } + + public static String getScheme(String uri) { return toUri(uri).getScheme(); } + public static String getHost(String uri) { return toUri(uri).getHost(); } + public static int getPort(String uri) { return toUri(uri).getPort(); } + public static String getPath(String uri) { return toUri(uri).getPath(); } + + public static String getHostUri(String uri) { + final URI u = toUri(uri); + return u.getScheme() + "://" + u.getHost(); + } + + /** + * getTLD("foo.bar.baz") == "baz" + * @param uri A URI that includes a host part + * @return the top-level domain + */ + public static String getTLD(String uri) { + final String parts[] = getHost(uri).split("\\."); + if (parts.length > 0) return parts[parts.length-1]; + throw new IllegalArgumentException("Invalid host in URI: "+uri); + } + + /** + * getRegisteredDomain("foo.bar.baz") == "bar.baz" + * @param uri A URI that includes a host part + * @return the "registered" domain, which includes the TLD and one level up. + */ + public static String getRegisteredDomain(String uri) { + final String host = getHost(uri); + final String parts[] = host.split("\\."); + switch (parts.length) { + case 0: throw new IllegalArgumentException("Invalid host: "+host); + case 1: return host; + default: return parts[parts.length-2] + "." + parts[parts.length-1]; + } + } + + public static String getFile(String uri) { + final String path = toUri(uri).getPath(); + final int last = path.lastIndexOf('/'); + if (last == -1 || last == path.length()-1) return null; + return path.substring(last+1); + } + + public static String getFileExt(String uri) { + final String path = toUri(uri).getPath(); + final int last = path.lastIndexOf('.'); + if (last == -1 || last == path.length()-1) return null; + return path.substring(last+1); + } + + public static boolean isHost(String uriString, String host) { + return !empty(uriString) && toUri(uriString).getHost().equals(host); + } + +} diff --git a/src/main/java/org/cobbzilla/util/http/main/HttpMain.java b/src/main/java/org/cobbzilla/util/http/main/HttpMain.java new file mode 100644 index 0000000..c9693d7 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/main/HttpMain.java @@ -0,0 +1,26 @@ +package org.cobbzilla.util.http.main; + +import org.cobbzilla.util.http.HttpRequestBean; +import org.cobbzilla.util.http.HttpResponseBean; +import org.cobbzilla.util.http.HttpUtil; +import org.cobbzilla.util.main.BaseMain; + +import static org.cobbzilla.util.daemon.ZillaRuntime.readStdin; +import static org.cobbzilla.util.json.JsonUtil.json; + +public class HttpMain extends BaseMain { + + public static void main (String[] args) { main(HttpMain.class, args); } + + @Override protected void run() throws Exception { + final HttpRequestBean request = json(readStdin(), HttpRequestBean.class); + if (request == null) die("nothing read from stdin"); + final HttpResponseBean response = HttpUtil.getResponse(request); + if (response.isOk()) { + out(response.getEntityString()); + } else { + err(json(response.toMap())); + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/http/main/HttpMainOptions.java b/src/main/java/org/cobbzilla/util/http/main/HttpMainOptions.java new file mode 100644 index 0000000..913448a --- /dev/null +++ b/src/main/java/org/cobbzilla/util/http/main/HttpMainOptions.java @@ -0,0 +1,5 @@ +package org.cobbzilla.util.http.main; + +import org.cobbzilla.util.main.BaseMainOptions; + +public class HttpMainOptions extends BaseMainOptions {} diff --git a/src/main/java/org/cobbzilla/util/io/BufferedFilesystemWatcher.java b/src/main/java/org/cobbzilla/util/io/BufferedFilesystemWatcher.java new file mode 100644 index 0000000..4317fd8 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/BufferedFilesystemWatcher.java @@ -0,0 +1,106 @@ +package org.cobbzilla.util.io; + +import lombok.Getter; +import lombok.ToString; +import org.cobbzilla.util.collection.InspectCollection; +import org.cobbzilla.util.system.Sleep; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.WatchEvent; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; + +import static org.cobbzilla.util.daemon.ZillaRuntime.now; +import static org.cobbzilla.util.daemon.ZillaRuntime.terminate; +import static org.cobbzilla.util.io.FileUtil.abs; + +/** + * Sometimes you just want to know that something changed, and you don't really care what. + * Extend this class and override the "fire" method. You will receive one callback when + * your timeout elapses, or if the buffer of events exceeds maxEvents. + */ +@ToString(callSuper=true, of={"timeout", "maxEvents"}) +public abstract class BufferedFilesystemWatcher extends FilesystemWatcher implements Closeable { + + @Getter private final long timeout; + @Getter private final int maxEvents; + + private final Thread monitor; + private long lastFlush; + private final Queue> buffer = new ConcurrentLinkedQueue<>(); + private BfsMonitor bfsMonitor; + + /** + * Called when some changes have occurred. + * This will be called if the number of events exceeds maxEvents, or if + * timeout milliseconds have elapsed since the last time it was called (any + * at least one event has occurred) + * @param events A collection of events. + */ + protected abstract void fire(List> events); + + public BufferedFilesystemWatcher(Path path, long timeout, int maxEvents) { + super(path); + bfsMonitor = new BfsMonitor(); + monitor = new Thread(bfsMonitor, "bfs-monitor("+abs(path)+")"); + monitor.setDaemon(true); + monitor.start(); + this.timeout = timeout; + this.maxEvents = maxEvents; + } + + public BufferedFilesystemWatcher(File path, long timeout, int maxEvents) { + this(path.toPath(), timeout, maxEvents); + } + + @Override public void close() throws IOException { + if (monitor != null) { + bfsMonitor.alive = false; + terminate(monitor, 2000); + } + super.close(); + } + + private boolean beenTooLong() { return now() - lastFlush > timeout; } + private boolean bufferTooBig() { return InspectCollection.isLargerThan(buffer, maxEvents); } + + private boolean shouldFlush() { return bufferTooBig() || (!buffer.isEmpty() && beenTooLong()); } + + @Override protected void handleEvent(WatchEvent event) { buffer.add(event); } + + private class BfsMonitor implements Runnable { + public volatile boolean alive = false; + @Override public void run() { + alive = true; + while (alive) { + Sleep.sleep(timeout / 10); + if (shouldFlush()) flush(); + } + } + } + + private synchronized void flush() { + // sanity check that we have not flushed recently + if (!shouldFlush()) return; + + // nothing to flush? + if (buffer.isEmpty()) return; + + final List> events = new ArrayList<>(buffer.size()); + while (!buffer.isEmpty()) { + events.add(buffer.poll()); + if (events.size() > maxEvents) { + fire(events); + events.clear(); + } + } + if (!events.isEmpty()) fire(events); + lastFlush = now(); + } + +} diff --git a/src/main/java/org/cobbzilla/util/io/ByteLimitedInputStream.java b/src/main/java/org/cobbzilla/util/io/ByteLimitedInputStream.java new file mode 100644 index 0000000..af80762 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/ByteLimitedInputStream.java @@ -0,0 +1,50 @@ +package org.cobbzilla.util.io; + +import lombok.Delegate; +import lombok.Getter; + +import java.io.IOException; +import java.io.InputStream; + +public class ByteLimitedInputStream extends InputStream { + + @Delegate(excludes=BLISDelegateExcludes.class) private InputStream delegate; + + private interface BLISDelegateExcludes { + int read(byte[] b) throws IOException; + int read(byte[] b, int off, int len) throws IOException; + int read() throws IOException; + } + + @Getter private long count = 0; + @Getter private long limit; + + public double getPercentDone () { return ((double) count) / ((double) limit); } + + public ByteLimitedInputStream (InputStream in, long limit) { + this.delegate = in; + this.limit = limit; + } + + @Override public int read(byte[] b) throws IOException { + if (count >= limit) return -1; + final int read = delegate.read(b); + if (read != -1) count += read; + return read; + } + + @Override public int read(byte[] b, int off, int len) throws IOException { + if (count >= limit) return -1; + final int read = delegate.read(b, off, len); + if (read != -1) count += read; + return read; + } + + @Override public int read() throws IOException { + if (count >= limit) return -1; + final int read = delegate.read(); + if (read != -1) count++; + return read; + } + +} diff --git a/src/main/java/org/cobbzilla/util/io/CloseOnExit.java b/src/main/java/org/cobbzilla/util/io/CloseOnExit.java new file mode 100644 index 0000000..2d3a5f6 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/CloseOnExit.java @@ -0,0 +1,34 @@ +package org.cobbzilla.util.io; + +import lombok.extern.slf4j.Slf4j; + +import java.io.Closeable; +import java.util.ArrayList; +import java.util.List; + +@Slf4j +public class CloseOnExit implements Runnable { + + private static List closeables = new ArrayList<>(); + + private CloseOnExit () {} + + static { + Runtime.getRuntime().addShutdownHook(new Thread(new CloseOnExit())); + } + + @Override public void run() { + if (closeables != null) { + for (Closeable c : closeables) { + try { + c.close(); + } catch (Exception e) { + log.error("Error closing: " + c + ": " + e, e); + } + } + } + } + + public static void add(Closeable closeable) { closeables.add(closeable); } + +} diff --git a/src/main/java/org/cobbzilla/util/io/CompositeBufferedFilesystemWatcher.java b/src/main/java/org/cobbzilla/util/io/CompositeBufferedFilesystemWatcher.java new file mode 100644 index 0000000..3316bb4 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/CompositeBufferedFilesystemWatcher.java @@ -0,0 +1,49 @@ +package org.cobbzilla.util.io; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.ToString; + +import java.io.File; +import java.nio.file.Path; +import java.nio.file.WatchEvent; +import java.util.Collection; +import java.util.List; + +@AllArgsConstructor @ToString(callSuper=true, of={"timeout", "maxEvents"}) +public abstract class CompositeBufferedFilesystemWatcher extends CompositeFilesystemWatcher { + + @Getter private final long timeout; + @Getter private final int maxEvents; + + public CompositeBufferedFilesystemWatcher (long timeout, int maxEvents, File[] paths) { + this(timeout, maxEvents); + addAll(paths); + } + + public CompositeBufferedFilesystemWatcher (long timeout, int maxEvents, String[] paths) { + this(timeout, maxEvents); + addAll(paths); + } + + public CompositeBufferedFilesystemWatcher (long timeout, int maxEvents, Path[] paths) { + this(timeout, maxEvents); + addAll(paths); + } + + public CompositeBufferedFilesystemWatcher (long timeout, int maxEvents, Collection things) { + this(timeout, maxEvents); + addAll(things); + } + + public abstract void fire(List> events); + private void _fire(List> events) { fire(events); } + + @Override protected BufferedFilesystemWatcher newWatcher(Path path) { + return new BufferedFilesystemWatcher(path, timeout, maxEvents) { + @Override protected void fire(List> events) { + _fire(events); + } + }; + } +} diff --git a/src/main/java/org/cobbzilla/util/io/CompositeFilesystemWatcher.java b/src/main/java/org/cobbzilla/util/io/CompositeFilesystemWatcher.java new file mode 100644 index 0000000..3f4bd03 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/CompositeFilesystemWatcher.java @@ -0,0 +1,80 @@ +package org.cobbzilla.util.io; + +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.string.StringUtil; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static org.cobbzilla.util.reflect.ReflectionUtil.getFirstTypeParam; + +@Slf4j +public abstract class CompositeFilesystemWatcher implements Closeable { + + private Map watchers = new ConcurrentHashMap<>(); + + @Override public void close() throws IOException { + Map copy = watchers; + watchers = null; + for (T watcher : copy.values()) { + watcher.close(); + } + } + + public List pathsWatching () { return new ArrayList<>(watchers.keySet()); } + + public List dirsWatching() { + List paths = pathsWatching(); + List dirs = new ArrayList<>(paths.size()); + for (Path p : paths) dirs.add(p.toFile()); + return dirs; + } + + public boolean isEmpty() { return watchers.isEmpty(); } + + protected abstract T newWatcher(Path path); + + public void add (String path) { add(new File(path)); } + + public void add (File path) { add(path.toPath()); } + + public void add (Path path) { + final T watcher = newWatcher(path); + final T old = watchers.remove(path); + if (old != null) { + log.warn("Replacing old watcher ("+old+") with new one: "+watcher); + old.stop(); + } + watcher.start(); + watchers.put(path, watcher); + } + + public void addAll(File[] paths) { if (paths != null) for (File p : paths) add(p); } + public void addAll(Path[] paths) { if (paths != null) for (Path p : paths) add(p); } + public void addAll(String[] paths) { if (paths != null) for (String p : paths) add(new File(p)); } + + public void addAll(Collection things) { + if (!things.isEmpty()) { + final Class clazz = things.iterator().next().getClass(); + if (clazz.equals(File.class)) { + addAll((File[]) things.toArray(new File[things.size()])); + } else if (clazz.equals(Path.class)) { + addAll((Path[]) things.toArray(new Path[things.size()])); + } else if (clazz.equals(String.class)) { + addAll((String[]) things.toArray(new String[things.size()])); + } + } + } + + @Override public String toString() { + return "CompositeFilesystemWatcher<"+getFirstTypeParam(getClass()).getName()+">{" + + "paths=" + StringUtil.toString(watchers.keySet(), " ") + "}"; + } +} diff --git a/src/main/java/org/cobbzilla/util/io/DamperedCompositeBufferedFilesystemWatcher.java b/src/main/java/org/cobbzilla/util/io/DamperedCompositeBufferedFilesystemWatcher.java new file mode 100644 index 0000000..6c868b4 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/DamperedCompositeBufferedFilesystemWatcher.java @@ -0,0 +1,100 @@ +package org.cobbzilla.util.io; + +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.nio.file.Path; +import java.nio.file.WatchEvent; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import static org.cobbzilla.util.system.Sleep.nap; + +@Slf4j +public abstract class DamperedCompositeBufferedFilesystemWatcher extends CompositeBufferedFilesystemWatcher { + + private final AtomicLong damper = new AtomicLong(0); + private final AtomicReference damperThread = new AtomicReference<>(); + private final AtomicReference>> buffer = new AtomicReference<>(); + + protected void init(long damperDuration, int maxEvents) { + + this.damper.set(damperDuration); + this.buffer.set(new ArrayList>(maxEvents*10)); + + this.damperThread.set(new Thread(new Runnable() { + @Override public void run() { + log.debug(status()+" starting, sleeping for a while until there is some activity"); + //noinspection InfiniteLoopStatement + while (true) { + if (!nap(TimeUnit.HOURS.toMillis(4), "waiting for filesystem watcher trigger to fire")) { + // we were interrupted. sleep for the damper time + while (!nap(damper.get())) { + // there was more activity, go back to sleep + log.debug(status()+" more activity while napping for damper, trying again"); + } + // we successfully napped without being interrupted! fire the big trigger + log.debug(status()+": napped successfully, calling fire"); + List> events; + synchronized (buffer) { + events = new ArrayList<>(buffer.get()); + buffer.get().clear(); + } + uber_fire(events); + } + log.debug(status()+" just fired, going back to sleep for a while until there is some more activity"); + } + } + })); + damperThread.get().setDaemon(true); + damperThread.get().start(); + } + + protected String status() { synchronized (buffer) { return "[" + buffer.get().size() + " events]"; } } + + /** + * Called when the thing finally really fires. + * @param events + */ + public abstract void uber_fire(List> events); + + @Override public void fire(List> events) { + log.debug(status()+": fire adding "+events.size()+" events..."); + synchronized (buffer) { + buffer.get().addAll(events); + } + synchronized (damperThread.get()) { + damperThread.get().interrupt(); + } + } + + public DamperedCompositeBufferedFilesystemWatcher(long timeout, int maxEvents, long damper) { + super(timeout, maxEvents); + init(damper, maxEvents); + } + + public DamperedCompositeBufferedFilesystemWatcher(long timeout, int maxEvents, File[] paths, long damper) { + super(timeout, maxEvents, paths); + init(damper, maxEvents); + } + + public DamperedCompositeBufferedFilesystemWatcher(long timeout, int maxEvents, String[] paths, long damper) { + super(timeout, maxEvents, paths); + init(damper, maxEvents); + } + + public DamperedCompositeBufferedFilesystemWatcher(long timeout, int maxEvents, Path[] paths, long damper) { + super(timeout, maxEvents, paths); + init(damper, maxEvents); + } + + public DamperedCompositeBufferedFilesystemWatcher(long timeout, int maxEvents, Collection things, long damper) { + super(timeout, maxEvents, things); + init(damper, maxEvents); + } + +} diff --git a/src/main/java/org/cobbzilla/util/io/Decompressors.java b/src/main/java/org/cobbzilla/util/io/Decompressors.java new file mode 100644 index 0000000..8afd8bb --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/Decompressors.java @@ -0,0 +1,80 @@ +package org.cobbzilla.util.io; + +import lombok.Cleanup; + +import java.io.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.io.Tarball.isTarball; + +// zip-related code adapted from: https://stackoverflow.com/a/10634536/1251543 +public class Decompressors { + + public static TempDir unroll (File infile) throws Exception { + if (isTarball(infile)) { + return Tarball.unroll(infile); + } else if (isZipFile(infile.getName())) { + final TempDir tempDir = new TempDir(); + extract(infile, tempDir); + return tempDir; + } else { + return die("unroll: unsupported file: "+infile); + } + } + + public static boolean isZipFile(String name) { + return name.toLowerCase().endsWith(".zip"); + } + + public static boolean isDecompressible(File file) { return isDecompressible(file.getName()); } + + public static boolean isDecompressible(String name) { + return isTarball(name) || isZipFile(name); + } + + private static void extractFile(ZipInputStream in, File outdir, String name) throws IOException { + @Cleanup final FileOutputStream out = new FileOutputStream(new File(outdir, name)); + StreamUtil.copyLarge(in, out); + } + + private static void mkdirs(File outdir, String path) { + final File d = new File(outdir, path); + if (!d.exists() && !d.mkdirs()) die("mkdirs("+abs(outdir)+", "+path+"): error creating "+abs(d)); + } + + private static String dirpart(String name) { + final int s = name.lastIndexOf( File.separatorChar ); + return s == -1 ? null : name.substring( 0, s ); + } + + /*** + * Extract zipfile to outdir with complete directory structure + * @param zipfile Input .zip file + * @param outdir Output directory + */ + public static void extract(File zipfile, File outdir) throws IOException { + @Cleanup final ZipInputStream zin = new ZipInputStream(new FileInputStream(zipfile)); + ZipEntry entry; + String name, dir; + while ((entry = zin.getNextEntry()) != null) { + name = entry.getName(); + if (entry.isDirectory()) { + mkdirs(outdir,name); + continue; + } + /* this part is necessary because file entry can come before + * directory entry where is file located + * i.e.: + * /foo/foo.txt + * /foo/ + */ + dir = dirpart(name); + if (dir != null) mkdirs(outdir,dir); + + extractFile(zin, outdir, name); + } + } +} diff --git a/src/main/java/org/cobbzilla/util/io/DeleteOnExit.java b/src/main/java/org/cobbzilla/util/io/DeleteOnExit.java new file mode 100644 index 0000000..75df106 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/DeleteOnExit.java @@ -0,0 +1,41 @@ +package org.cobbzilla.util.io; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@Slf4j +public class DeleteOnExit implements Runnable { + + private static List paths = new ArrayList<>(); + + private DeleteOnExit () {} + + static { + Runtime.getRuntime().addShutdownHook(new Thread(new DeleteOnExit())); + } + + @Override public void run() { + for (File path : paths) { + if (!path.exists()) return; + if (path.isDirectory()) { + try { + FileUtils.deleteDirectory(path); + } catch (IOException e) { + log.warn("FileUtil.deleteOnExit: error deleting path=" + path + ": " + e, e); + } + } else { + if (!path.delete()) { + log.warn("FileUtil.deleteOnExit: error deleting path=" + path); + } + } + } + } + + public static void add(File path) { paths.add(path); } + +} diff --git a/src/main/java/org/cobbzilla/util/io/DirFilter.java b/src/main/java/org/cobbzilla/util/io/DirFilter.java new file mode 100644 index 0000000..abd8251 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/DirFilter.java @@ -0,0 +1,28 @@ +package org.cobbzilla.util.io; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.io.File; +import java.io.FileFilter; +import java.util.regex.Pattern; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +@NoArgsConstructor @AllArgsConstructor +public class DirFilter implements FileFilter { + + public static final DirFilter instance = new DirFilter(); + + @Getter @Setter private String regex; + + @Getter(lazy=true) private final Pattern pattern = initPattern(); + private Pattern initPattern() { return Pattern.compile(regex); } + + @Override public boolean accept(File pathname) { + return pathname.isDirectory() && (empty(regex) || getPattern().matcher(pathname.getName()).matches()); + } + +} diff --git a/src/main/java/org/cobbzilla/util/io/FileResolver.java b/src/main/java/org/cobbzilla/util/io/FileResolver.java new file mode 100644 index 0000000..461e0bb --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/FileResolver.java @@ -0,0 +1,9 @@ +package org.cobbzilla.util.io; + +import java.io.File; + +public interface FileResolver { + + File resolve (String path); + +} diff --git a/src/main/java/org/cobbzilla/util/io/FileSuffixFilter.java b/src/main/java/org/cobbzilla/util/io/FileSuffixFilter.java new file mode 100644 index 0000000..fc8d7f3 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/FileSuffixFilter.java @@ -0,0 +1,17 @@ +package org.cobbzilla.util.io; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + +import java.io.File; +import java.io.FileFilter; + +@AllArgsConstructor +public class FileSuffixFilter implements FileFilter { + + @Getter @Setter private String suffix; + + @Override public boolean accept(File pathname) { return pathname.getName().endsWith(suffix); } + +} diff --git a/src/main/java/org/cobbzilla/util/io/FileUtil.java b/src/main/java/org/cobbzilla/util/io/FileUtil.java new file mode 100644 index 0000000..29ae9a5 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/FileUtil.java @@ -0,0 +1,626 @@ +package org.cobbzilla.util.io; + +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.cobbzilla.util.string.Base64; + +import java.io.*; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.apache.commons.lang3.StringUtils.chop; +import static org.cobbzilla.util.daemon.ZillaRuntime.*; +import static org.cobbzilla.util.io.TempDir.quickTemp; +import static org.cobbzilla.util.system.CommandShell.chmod; + +@Slf4j +public class FileUtil { + + public static final File DEFAULT_TEMPDIR = new File(System.getProperty("java.io.tmpdir")); + private static final File[] EMPTY_ARRAY = {}; + public static final String sep = File.separator; + + public static String defaultTempDir = System.getProperty("user.home") + "/tmp/zilla"; + + public static boolean isReadableNonEmptyFile (File f) { + return f != null && f.exists() && f.canRead() && f.length() > 0; + } + + /** + * Iterate through given paths and return File object for the first found path and filename combination which + * results with an existing readable and non empty file. + * + * @param paths List of paths to look for the file in + * @param filename The name of the file. + * @return First found file or null. + */ + public static File firstFoundFile(Collection paths, String filename) { + if (!empty(paths)) { + for (String path : paths) { + File f = new File(path, filename); + if (isReadableNonEmptyFile(f)) return f; + } + } + // Finally try from withing the current folder: + File f = new File(filename); + if (isReadableNonEmptyFile(f)) return f; + + return null; + } + + public static File[] list(File dir) { + final File[] files = dir.listFiles(); + if (files == null) return EMPTY_ARRAY; + return files; + } + + public static File[] listFiles(File dir) { + final File[] files = dir.listFiles(RegularFileFilter.instance); + if (files == null) return EMPTY_ARRAY; + return files; + } + + public static File[] listFiles(File dir, FileFilter filter) { + final File[] files = dir.listFiles(filter); + if (files == null) return EMPTY_ARRAY; + return files; + } + + public static List listFilesRecursively(File dir) { return listFilesRecursively(dir, null); } + + public static List listFilesRecursively(File dir, FileFilter filter) { + final List files = new ArrayList<>(); + _listRecurse(files, dir, filter); + return files; + } + + private static List _listRecurse(List results, File dir, FileFilter filter) { + final File[] files = filter == null ? dir.listFiles() : dir.listFiles(filter); + if (files == null) return results; + results.addAll(Arrays.asList(files)); + + final File[] subdirs = listDirs(dir); + for (File subdir : subdirs) { + _listRecurse(results, subdir, filter); + } + return results; + } + + public static File[] listDirs(File dir) { + return listDirs(dir, null); + } + + public static File[] listDirs(File dir, String regex) { + final File[] files = dir.listFiles(empty(regex) ? DirFilter.instance : new DirFilter(regex)); + if (files == null) return EMPTY_ARRAY; + return files; + } + + public static String chopSuffix(String path) { + if (path == null) return null; + final int lastDot = path.lastIndexOf('.'); + if (lastDot == -1 || lastDot == path.length()-1) return path; + return path.substring(0, lastDot); + } + + public static File createTempDir(String prefix) throws IOException { + return createTempDir(DEFAULT_TEMPDIR, prefix); + } + + public static File createTempDir(File parentDir, String prefix) throws IOException { + final Path parent = FileSystems.getDefault().getPath(abs(parentDir)); + return new File(Files.createTempDirectory(parent, prefix).toAbsolutePath().toString()); + } + + public static File createTempDirOrDie(String prefix) { + return createTempDirOrDie(DEFAULT_TEMPDIR, prefix); + } + + public static File createTempDirOrDie(File parentDir, String prefix) { + try { + return createTempDir(parentDir, prefix); + } catch (IOException e) { + return die("createTempDirOrDie: error creating directory with prefix="+abs(parentDir)+"/"+prefix+": "+e, e); + } + } + + public static void writeResourceToFile(String resourcePath, File outFile, Class clazz) throws IOException { + if (!outFile.getParentFile().exists() || !outFile.getParentFile().canWrite() || (outFile.exists() && !outFile.canWrite())) { + throw new IllegalArgumentException("outFile is not writeable: "+abs(outFile)); + } + try (InputStream in = clazz.getClassLoader().getResourceAsStream(resourcePath); + OutputStream out = new FileOutputStream(outFile)) { + if (in == null) throw new IllegalArgumentException("null data at resourcePath: "+resourcePath); + IOUtils.copy(in, out); + } + } + + public static List loadResourceAsStringListOrDie(String resourcePath, Class clazz) { + try { + return loadResourceAsStringList(resourcePath, clazz); + } catch (IOException e) { + throw new IllegalArgumentException("loadResourceAsStringList error: "+e, e); + } + } + + public static List loadResourceAsStringList(String resourcePath, Class clazz) throws IOException { + @Cleanup final Reader reader = StreamUtil.loadResourceAsReader(resourcePath, clazz); + return toStringList(reader); + } + + public static List toStringList(String f) throws IOException { + return toStringList(new File(f)); + } + + public static List toStringList(File f) throws IOException { + @Cleanup final Reader reader = new FileReader(f); + return toStringList(reader); + } + + public static List toStringList(Reader reader) throws IOException { + final List strings = new ArrayList<>(); + try (BufferedReader r = new BufferedReader(reader)) { + String line; + while ((line = r.readLine()) != null) { + strings.add(line.trim()); + } + } + return strings; + } + + public static File toFile (List lines) throws IOException { + final File temp = File.createTempFile(FileUtil.class.getSimpleName()+".toFile", "tmp", getDefaultTempDir()); + try (BufferedWriter writer = new BufferedWriter(new FileWriter(temp))) { + for (String line : lines) { + writer.write(line+"\n"); + } + } + return temp; + } + + public static String toStringOrDie (String f) { + return toStringOrDie(new File(f)); + } + + public static String toStringOrDie (File f) { + try { + return toString(f); + } catch (FileNotFoundException e) { + log.warn("toStringOrDie: returning null; file not found: "+abs(f)); + return null; + } catch (IOException e) { + final String path = f == null ? "null" : abs(f); + throw new IllegalArgumentException("Error reading file ("+ path +"): "+e, e); + } + } + + public static String toString (String f) throws IOException { + return toString(new File(f)); + } + + public static String toString (File f) throws IOException { + if (f == null || !f.exists()) return null; + final StringWriter writer = new StringWriter(); + try (Reader r = new FileReader(f)) { + IOUtils.copy(r, writer); + } + return writer.toString(); + } + + public static byte[] toBytes (File f) throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (InputStream in = new FileInputStream(f)) { + IOUtils.copy(in, out); + } + return out.toByteArray(); + } + + public static Properties toPropertiesOrDie (String f) { + return toPropertiesOrDie(new File(f)); + } + + private static Properties toPropertiesOrDie(File f) { + try { + return toProperties(f); + } catch (IOException e) { + final String path = f == null ? "null" : abs(f); + throw new IllegalArgumentException("Error reading properties file ("+ path +"): "+e, e); + } + } + + public static Properties toProperties (String f) throws IOException { + return toProperties(new File(f)); + } + + public static Properties toProperties (File f) throws IOException { + final Properties props = new Properties(); + try (InputStream in = new FileInputStream(f)) { + props.load(in); + } + return props; + } + + public static Properties resourceToPropertiesOrDie (String path, Class clazz) { + try { + return resourceToProperties(path, clazz); + } catch (IOException e) { + throw new IllegalArgumentException("Error reading resource ("+ path +"): "+e, e); + } + } + + public static Properties resourceToProperties (String path, Class clazz) throws IOException { + final Properties props = new Properties(); + try (InputStream in = StreamUtil.loadResourceAsStream(path, clazz)) { + props.load(in); + } + return props; + } + + public static File secureFile (File dir, String name, String data) { + final File f = secureWritableFile(dir, name, data); + chmod(f, "400"); + return f; + } + + public static File secureWritableFile(File dir, String name, String data) { + final File f = new File(dir, name); + if (f.exists() && !f.delete()) return die("secureWritableFile: error deleting file: "+abs(f)); + touch(f); + chmod(f, "600"); + toFileOrDie(f, data); + return f; + } + + public static File secureFileB64 (File dir, String name, String data) { + final File f = secureWritableFile(dir, name, ""); + try (FileOutputStream out = new FileOutputStream(f)) { + IOUtils.copyLarge(new ByteArrayInputStream(Base64.decode(data)), out); + } catch (Exception e) { + return die("secureFileB64: "+e); + } + chmod(f, "400"); + return f; + } + + public static File toFileOrDie (String file, String data) { + return toFileOrDie(new File(file), data); + } + + public static File toFileOrDie (File file, String data) { + return toFileOrDie(file, data, false); + } + + public static File toFileOrDie(File file, String data, boolean append) { + try { + return toFile(file, data, append); + } catch (IOException e) { + String path = (file == null) ? "null" : abs(file); + return die("toFileOrDie: error writing to file: "+path+": "+e, e); + } + } + + public static File toFile (String data) { + try { + return toFile(temp(".tmp"), data, false); + } catch (IOException e) { + return die("toFile: error writing data to temp file: "+e, e); + } + } + public static File toTempFile (String data, String ext) { + return toFileOrDie(temp(ext), data, false); + } + + public static File toFile (String file, String data) throws IOException { + return toFile(new File(file), data); + } + + public static File toFileOrDie(File file, InputStream in) { + try { return toFile(file, in); } catch (Exception e) { return die("toFileOrDie: "+e, e); } + } + + public static File toFile(File file, InputStream in) throws IOException { + try (OutputStream out = new FileOutputStream(file)) { + IOUtils.copyLarge(in, out); + } + return file; + } + + public static File toFile(File file, String data) throws IOException { + return toFile(file, data, false); + } + + public static File appendFile(File file, String data) throws IOException { + return toFile(file, data, true); + } + + public static File toFile(File file, String data, boolean append) throws IOException { + if (!ensureDirExists(file.getParentFile())) { + throw new IOException("Error creating directory: "+file.getParentFile()); + } + try (OutputStream out = new FileOutputStream(file, append)) { + IOUtils.copy(new ByteArrayInputStream(data.getBytes()), out); + } + return file; + } + + public static void renameOrDie (File from, File to) { + if (!from.renameTo(to)) die("Error renaming "+abs(from)+" -> "+abs(to)); + } + + public static void writeString (File target, String data) throws IOException { + try (FileWriter w = new FileWriter(target)) { + w.write(data); + } + } + + public static void writeStringOrDie (File target, String data) { + try { + writeString(target, data); + } catch (IOException e) { + die("Error writing to file ("+abs(target)+"): "+e, e); + } + } + + public static void touch (String file) { touch(new File(file)); } + + public static void touch (File file) { + if (!file.exists()) { + try (OutputStream out = new FileOutputStream(file)) { + } catch (Exception e) { + die("touch: "+e, e); + } + } + file.setLastModified(now()); + } + + public static Path path(File f) { + return FileSystems.getDefault().getPath(abs(f)); + } + + public static boolean isSymlink(File file) { + return Files.isSymbolicLink(path(file)); + } + + public static String toStringExcludingLines(File file, String prefix) throws IOException { + final StringBuilder sb = new StringBuilder(); + try (BufferedReader reader = new BufferedReader(new FileReader(file))) { + String line; + while ((line = reader.readLine()) != null) { + if (!line.trim().startsWith(prefix)) sb.append(line).append("\n"); + } + } + return sb.toString(); + } + + public static String dirname(String path) { + if (empty(path)) throw new NullPointerException("dirname: path was empty"); + final int pos = path.lastIndexOf('/'); + if (pos == -1) return "."; + if (path.endsWith("/")) path = chop(path); + return path.substring(0, pos); + } + + public static String basename(String path) { + if (empty(path)) throw new NullPointerException("basename: path was empty"); + final int pos = path.lastIndexOf('/'); + if (pos == -1) return path; + if (pos == path.length()-1) throw new IllegalArgumentException("basename: invalid path: "+path); + return path.substring(pos + 1); + } + + // quick alias for getting an absolute path + public static String abs(File path) { + try { + return path == null ? "null" : path.getCanonicalPath(); + } catch (IOException e) { + log.warn("abs("+path.getAbsolutePath()+"): "+e); + return path.getAbsolutePath(); + } + } + public static String abs(Path path) { return path == null ? "null" : abs(path.toFile()); } + public static String abs(String path) { return path == null ? "null" : abs(new File(path)); } + + public static File mkdirOrDie(String dir) { return mkdirOrDie(new File(dir)); } + + public static File mkdirOrDie(File dir) { + if (!dir.exists() && !dir.mkdirs()) { + if (!dir.exists()) { + final String msg = "mkdirOrDie: error creating: " + abs(dir); + log.error(msg); + die(msg); + } + } + assertIsDir(dir); + return dir; + } + + public static boolean ensureDirExists(File dir) { + if (dir == null) { + log.error("ensureDirExists: null as directory is not acceptable"); + return false; + } + if (dir.exists() && dir.isDirectory()) return true; + if (!dir.exists() && !dir.mkdirs()) { + log.error("ensureDirExists: error creating: " + abs(dir)); + return false; + } + if (!dir.isDirectory()) { + log.error("ensureDirExists: not a directory: " + abs(dir)); + return false; + } + return true; + } + + public static void assertIsDir(File dir) { + if (!dir.isDirectory()) { + final String msg = "assertIsDir: not a dir: " + abs(dir); + log.error(msg); + throw new IllegalArgumentException(msg); + } + } + + public static String extension(File f) { return extension(abs(f)); } + + public static String extension(String name) { + final int lastDot = name.lastIndexOf('.'); + if (lastDot == -1) return ""; + return name.substring(lastDot); + } + + public static String extensionOrName(String name) { + final String ext = extension(name); + return empty(ext) ? name : ext; + } + + public static String removeExtension(File f, String ext) { + return f.getName().substring(0, f.getName().length() - ext.length()); + } + + /** + * @param dir The directory to search + * @return The most recently modified file, or null if the dir does not exist, is not a directory, or does not contain any files + */ + public static File mostRecentFile(File dir) { + if (!dir.exists()) return null; + File newest = null; + for (File file : list(dir)) { + if (file.isDirectory()) { + file = mostRecentFile(file); + if (file == null) continue; + } + if (file.isFile()) { + if (newest == null) { + newest = file; + } else if (file.lastModified() > newest.lastModified()) { + newest = file; + } + } + } + return newest; + } + + public static boolean mostRecentFileIsNewerThan(File dir, long time) { + final File newest = mostRecentFile(dir); + return newest != null && newest.lastModified() > time; + } + + public static File mkHomeDir(String subDir) { + + final String homeDir = getUserHomeDir(); + if (empty(homeDir)) die("mkHomeDir: System.getProperty(\"user.home\") returned nothing useful: "+homeDir); + + if (!subDir.startsWith("/")) subDir = "/" + subDir; + + return mkdirOrDie(new File(homeDir + subDir)); + } + + public static String getUserHomeDir() { + // todo: ensure this works correctly in sandboxed-environments (mac app store) + return System.getProperty("user.home"); + } + + public static void copyFile(File from, File to) { + try { + if (!to.getParentFile().exists() && !to.getParentFile().mkdirs()) { + if (!to.getParentFile().exists()) { + die("Error creating parent dir: " + abs(to.getParentFile())); + } + } + FileUtils.copyFile(from, to); + } catch (IOException e) { + die("copyFile: "+e, e); + } + } + + public static void deleteOrDie(File f) { + if (f == null) return; + if (f.exists()) { + FileUtils.deleteQuietly(f); + if (f.exists()) die("delete: Error deleting: "+abs(f)); + } + } + + public static int countFilesWithName(File dir, String name) { + int count = 0; + final File[] files = dir.listFiles(); + if (files != null) { + for (File f : files) { + if (f.isDirectory()) count += countFilesWithName(f, name); + else if (f.getName().equals(name)) count++; + } + } + return count; + } + + public static final long DEFAULT_KILL_AFTER = TimeUnit.MINUTES.toMillis(5); + + public static File bzip2(File f) throws IOException { return bzip2(f, DEFAULT_KILL_AFTER); } + + public static File bzip2(File f, long killAfter) throws IOException { + final File temp = quickTemp(killAfter); + try (OutputStream bzout = new BZip2CompressorOutputStream(new FileOutputStream(temp))) { + try (InputStream in = new FileInputStream(f)) { + IOUtils.copyLarge(in, bzout); + } + } + return temp; + } + + public static File bzip2(InputStream fileStream) throws IOException { + return bzip2(fileStream, DEFAULT_KILL_AFTER); + } + + public static File bzip2(InputStream fileStream, long killAfter) throws IOException { + return bzip2(FileUtil.toFile(quickTemp(killAfter), fileStream)); + } + + public static File symlink (File link, File target) throws IOException { + return Files.createSymbolicLink(link.toPath(), target.toPath()).toFile(); + } + + public static File temp (String suffix) { return temp("temp-", suffix); } + + public static File temp (String prefix, String suffix) { + try { + return File.createTempFile(prefix, suffix, getDefaultTempDir()); + } catch (IOException e) { + return die("temp: "+e, e); + } + } + + public static File getDefaultTempDir() { return mkdirOrDie(defaultTempDir); } + + public static File temp (String prefix, String suffix, File dir) { + try { + return File.createTempFile(prefix, suffix, dir); + } catch (IOException e) { + return die("temp: "+e, e); + } + } + + public static File findFile(File dir, Pattern pattern) { + final AtomicReference found = new AtomicReference<>(); + new FilesystemWalker() + .withDir(dir) + .withVisitor(file -> { + if (found.get() != null) return; + final String path = abs(file); + final Matcher m = pattern.matcher(path); + if (m.find() && m.end() == path.length()) { + synchronized (found) { + if (found.get() == null) found.set(file); + } + } + }).walk(); + return found.get(); + } +} diff --git a/src/main/java/org/cobbzilla/util/io/FilenameSuffixFilter.java b/src/main/java/org/cobbzilla/util/io/FilenameSuffixFilter.java new file mode 100644 index 0000000..3f8586c --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/FilenameSuffixFilter.java @@ -0,0 +1,19 @@ +package org.cobbzilla.util.io; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + +import java.io.File; +import java.io.FilenameFilter; + +@AllArgsConstructor +public class FilenameSuffixFilter implements FilenameFilter { + + @Getter @Setter private String suffix; + + @Override public boolean accept(File dir, String name) { + return name.endsWith(suffix); + } + +} diff --git a/src/main/java/org/cobbzilla/util/io/FilesystemVisitor.java b/src/main/java/org/cobbzilla/util/io/FilesystemVisitor.java new file mode 100644 index 0000000..c35f84a --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/FilesystemVisitor.java @@ -0,0 +1,7 @@ +package org.cobbzilla.util.io; + +import java.io.File; + +public interface FilesystemVisitor { + void visit(File file); +} diff --git a/src/main/java/org/cobbzilla/util/io/FilesystemWalker.java b/src/main/java/org/cobbzilla/util/io/FilesystemWalker.java new file mode 100644 index 0000000..338d5c6 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/FilesystemWalker.java @@ -0,0 +1,112 @@ +package org.cobbzilla.util.io; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.daemon.AwaitResult; +import org.cobbzilla.util.string.StringUtil; + +import java.io.File; +import java.io.FileFilter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.cobbzilla.util.daemon.Await.awaitAll; +import static org.cobbzilla.util.daemon.DaemonThreadFactory.fixedPool; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.io.FileUtil.isSymlink; +import static org.cobbzilla.util.system.Sleep.sleep; +import static org.cobbzilla.util.time.TimeUtil.parseDuration; + +@Accessors(chain=true) @Slf4j +public class FilesystemWalker { + + @Getter private final List dirs = new ArrayList<>(); + @Getter private final List visitors = new ArrayList<>(); + @Getter @Setter private boolean includeSymlinks = true; + @Getter @Setter private boolean visitDirs = false; + @Getter @Setter private int threads = 5; + @Getter @Setter private int size = 1_000_000; + @Getter @Setter private long timeout = TimeUnit.MINUTES.toMillis(15); + @Getter @Setter private FileFilter filter; + @Getter @Setter private long sleepTime = TimeUnit.SECONDS.toMillis(5); + + public boolean hasFilter () { return filter != null; } + + public FilesystemWalker withDir (File dir) { dirs.add(dir); return this; } + public FilesystemWalker withDirs (List dirs) { this.dirs.addAll(dirs); return this; } + public FilesystemWalker withDirs (File[] dirs) { this.dirs.addAll(Arrays.asList(dirs)); return this; } + public FilesystemWalker withVisitor (FilesystemVisitor visitor) { visitors.add(visitor); return this; } + public FilesystemWalker withTimeoutDuration (String duration) { setTimeout(parseDuration(duration)); return this; } + + @Getter(lazy=true) private final ExecutorService pool = fixedPool(getThreads()); + @Getter(lazy=true) private final List> futures = new ArrayList<>(getSize()); + + public AwaitResult walk() { + for (File dir : dirs) fileJob(dir); + + // wait for number of futures to stop increasing + do { + final int lastNumFutures = numFutures(); + awaitFutures(); + if (numFutures() == lastNumFutures) break; + sleep(getSleepTime()); + } while (true); + return awaitFutures(); + } + + private AwaitResult awaitFutures() { + final AwaitResult result = awaitAll(getFutures(), getTimeout()); + if (!result.allSucceeded()) log.warn(StringUtil.toString(result.getFailures().values(), "\n---------")); + return result; + } + + private int numFutures () { + final List> futures = getFutures(); + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (futures) { return futures.size(); } + } + + private boolean fileJob(File f) { + final List> futures = getFutures(); + final Future future = getPool().submit(new FsWalker(f)); + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (futures) { return futures.add(future); } + } + + @AllArgsConstructor + private class FsWalker implements Runnable { + private final File file; + + @Override public void run() { + if (isSymlink(file) && !includeSymlinks) return; + + if (file.isFile()) { + // visit the file + visit(); + + } else if (file.isDirectory()) { + // should we visit directory entries? + if (visitDirs) visit(); + + // should we filter directory entries? + final File[] files = hasFilter() ? file.listFiles(filter) : file.listFiles(); + + // walk each entry in the directory + if (files != null) for (File f : files) fileJob(f); + + } else { + log.warn("unexpected file: neither file nor directory, skipping: "+abs(file)); + } + + } + + private void visit() { for (FilesystemVisitor visitor : getVisitors()) visitor.visit(file); } + } +} diff --git a/src/main/java/org/cobbzilla/util/io/FilesystemWatcher.java b/src/main/java/org/cobbzilla/util/io/FilesystemWatcher.java new file mode 100644 index 0000000..0f139f6 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/FilesystemWatcher.java @@ -0,0 +1,156 @@ +package org.cobbzilla.util.io; + +import lombok.Cleanup; +import lombok.Getter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.system.Sleep; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.nio.file.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.terminate; +import static org.cobbzilla.util.io.FileUtil.abs; + +@Slf4j @ToString(of={"path", "done"}) +public class FilesystemWatcher implements Runnable, Closeable { + + public static final long STOP_TIMEOUT = TimeUnit.SECONDS.toMillis(5); + + private final Thread thread = new Thread(this); + private final AtomicBoolean done = new AtomicBoolean(false); + @Getter private final Path path; + + public FilesystemWatcher(File path) { this.path = path.toPath(); } + public FilesystemWatcher(Path path) { this.path = path; } + + public synchronized void start () { + done.set(false); + thread.setDaemon(true); + thread.start(); + } + + public synchronized void stop () { + done.set(true); + terminate(thread, STOP_TIMEOUT); + } + + @Override public void close() throws IOException { stop(); } + + // print the events and the affected file + protected void handleEvent(WatchEvent event) { + + WatchEvent.Kind kind = event.kind(); + Path path = event.context() instanceof Path ? (Path) event.context() : null; + File file = path == null ? null : path.toFile(); + + if (file == null) { + log.warn("null path in event: "+event); + return; + } + + if (kind.equals(StandardWatchEventKinds.ENTRY_CREATE)) { + if (file.isDirectory()) { + onDirCreated(toFile(path)); + } else { + onFileCreated(toFile(path)); + } + } else if (kind.equals(StandardWatchEventKinds.ENTRY_DELETE)) { + if (file.isDirectory()) { + onDirDeleted(toFile(path)); + } else { + onFileDeleted(toFile(path)); + } + } else if (kind.equals(StandardWatchEventKinds.ENTRY_MODIFY)) { + if (file.isDirectory()) { + onDirModified(toFile(path)); + } else { + onFileModified(toFile(path)); + } + } + } + + protected void onDirCreated(File path) { log.info("dir created: "+ abs(path)); } + protected void onFileCreated(File path) { log.info("file created: "+ abs(path)); } + + protected void onDirModified(File path) { log.info("dir modified: "+ abs(path)); } + protected void onFileModified(File path) { log.info("file modified: "+ abs(path)); } + + protected void onDirDeleted(File path) { log.info("dir deleted: "+ abs(path)); } + protected void onFileDeleted(File path) { log.info("file deleted: "+ abs(path)); } + + public File toFile(Path p) { return new File(path.toFile(), p.toFile().getName()); } + + /** + * If the path does not exist, we cannot create the watch. But we can keep trying, and we do. + * @return how long to wait before retrying to create the watch, if the path didn't exist + */ + protected long getSleepWhileNotExists() { return 10_000; } + + /** + * If null is returned, the watcher will terminate on any unexpected Exception + * @return how long to sleep after some other unknown Exception (besides InterruptedException) occurs. + */ + protected Integer getSleepAfterUnexpectedError() { return 10_000; } + + @Override public void run() { + boolean logNotExists = true; + while (!done.get()) { + try { + log.info("Registering watch service on " + path); + @Cleanup final WatchService watchService = path.getFileSystem().newWatchService(); + path.register(watchService, + StandardWatchEventKinds.ENTRY_CREATE, + StandardWatchEventKinds.ENTRY_MODIFY, + StandardWatchEventKinds.ENTRY_DELETE); + logNotExists = true; + + // loop forever to watch directory + while (!done.get()) { + final WatchKey watchKey; + watchKey = watchService.take(); // this call is blocking until events are present + + // poll for file system events on the WatchKey + log.info("Waiting for FS events on " + path); + for (final WatchEvent event : watchKey.pollEvents()) { + log.info("Handling event: " + event.kind().name() + " " + event.context()); + handleEvent(event); + } + + // if the watched directed gets deleted, get out of run method + if (!watchKey.reset()) { + log.warn("watchKey could not be reset, perhaps path (" + path + ") was removed?"); + watchKey.cancel(); + watchService.close(); + break; + } + } + + } catch (InterruptedException e) { + die("watch thread interrupted, exiting: " + e, e); + + } catch (NoSuchFileException e) { + if (logNotExists) { + log.warn("watch dir does not exist, waiting for it to exist: " + e); + logNotExists = false; + } + Sleep.sleep(getSleepWhileNotExists(), "waiting for path to exist: " + abs(path)); + + } catch (Exception e) { + if (getSleepAfterUnexpectedError() != null) { + log.warn("error in watch thread, waiting to re-create the watch: " + e, e); + Sleep.sleep(getSleepAfterUnexpectedError()); + + } else { + die("error in watch thread, exiting: " + e, e); + } + } + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/io/JarTrimmer.java b/src/main/java/org/cobbzilla/util/io/JarTrimmer.java new file mode 100644 index 0000000..54f53d1 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/JarTrimmer.java @@ -0,0 +1,125 @@ +package org.cobbzilla.util.io; + +import lombok.Cleanup; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.cobbzilla.util.reflect.ReflectionUtil; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.util.*; +import java.util.function.Function; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +@Slf4j +public class JarTrimmer { + + public static final String CLASS_SUFFIX = ".class"; + @Getter private IncludeCount counter = new IncludeCount(""); + + public static class IncludeCount { + @Getter private int count; + + public int getTotalCount () { + int total = count; + for (IncludeCount count : subPaths.values()) total += count.getTotalCount(); + return total; + } + + @Getter private String path; + + @Getter private Map subPaths; + + public IncludeCount (String path) { + this.path = path; + this.count = 0; + this.subPaths = new HashMap<>(); + } + + public void incr() { count++; } + + public IncludeCount getCounter(String path) { + if (empty(path)) return this; + final int slashPos = path.indexOf('/'); + final boolean hasSlash = slashPos != -1; + final String part = hasSlash ? path.substring(0, slashPos) : path ; + final IncludeCount subCount = subPaths.computeIfAbsent(part, v -> new IncludeCount(part)); + return hasSlash ? subCount.getCounter(path.substring(slashPos+1)) : subCount; + } + } + + public IncludeCount trim (JarTrimmerConfig config) throws Exception { + + final JarFile jar = new JarFile(config.getInJar()); + + // walk all class resources in jar file, track location/count of required classes + processJar(jar, jarEntry -> { + final String name = jarEntry.getName(); + if (name.endsWith(CLASS_SUFFIX)) { + if (config.required(name)) counter.getCounter(toPath(name)).incr(); + } + return null; + }); + + // Level 1: any packages that do not contain ANY required classes will not be included in the output jar + final File temp = FileUtil.temp(".jar"); + @Cleanup final JarOutputStream jarOut = new JarOutputStream(new FileOutputStream(temp)); + final Set dirsCreated = new HashSet<>(); + processJar(jar, jarEntry -> { + final String name = jarEntry.getName(); + if (shouldInclude(config, name)) { + try { + final String dir = name.contains("/") ? toPath(name) : null; + if (dir != null && !dirsCreated.contains(dir)) { + final JarEntry dirEntry = new JarEntry(dir +"/"); + dirEntry.setTime(jarEntry.getTime()); + jarOut.putNextEntry(dirEntry); + dirsCreated.add(dir); + } + final JarEntry outEntry = new JarEntry(jarEntry.getName()); + ReflectionUtil.copy(outEntry, jarEntry, new String[] {"method", "time", "size", "compressedSize", "crc"}); + jarOut.putNextEntry(outEntry); + @Cleanup final InputStream in = jar.getInputStream(jarEntry); + IOUtils.copy(in, jarOut); + + } catch (Exception e) { + return die("processJar: " + e, e); + } + } else if (!name.endsWith("/")) { + log.info("omitted: "+ name); + } + return null; + }); + + FileUtil.renameOrDie(temp, config.getOutJar()); + return counter; + } + + private boolean shouldInclude(JarTrimmerConfig config, String name) { + if (name.endsWith("/")) return false; + if (config.required(name)) return true; + return counter.getCounter(toPath(name)).getTotalCount() > 0; + } + + private void processJar (JarFile jar, Function func) { + final Enumeration enumeration = jar.entries(); + while (enumeration.hasMoreElements()) func.apply(enumeration.nextElement()); + } + + private String toPath(String jarEntryName) { + final int lastSlash = jarEntryName.lastIndexOf('/'); + return (lastSlash == -1 || lastSlash == jarEntryName.length()-1) ? jarEntryName : jarEntryName.substring(0, lastSlash); + } + + protected static String toClassName(String jarEntryName) { + return jarEntryName.substring(0, jarEntryName.length()-CLASS_SUFFIX.length()).replace("/", "."); + } + +} diff --git a/src/main/java/org/cobbzilla/util/io/JarTrimmerConfig.java b/src/main/java/org/cobbzilla/util/io/JarTrimmerConfig.java new file mode 100644 index 0000000..9c4eef4 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/JarTrimmerConfig.java @@ -0,0 +1,56 @@ +package org.cobbzilla.util.io; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; +import org.apache.commons.io.FileUtils; +import org.cobbzilla.util.collection.ArrayUtil; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import static org.cobbzilla.util.string.StringUtil.UTF8cs; + +@Accessors(chain=true) +public class JarTrimmerConfig { + + @Getter @Setter private File inJar; + @Setter private File outJar; + public File getOutJar () { return outJar != null ? outJar : inJar; } + + @Getter @Setter private String[] requiredClasses; + @JsonIgnore @Getter(lazy=true) private final Set requiredClassSet = new HashSet<>(Arrays.asList(getRequiredClasses())); + + public JarTrimmerConfig setRequiredClassesFromFile (File f) throws IOException { + requiredClasses = FileUtils.readLines(f, UTF8cs).toArray(new String[0]); + return this; + } + + @Getter @Setter private String[] requiredPrefixes = new String[] { "META-INF", "WEB-INF" }; + public JarTrimmerConfig requirePrefix(String prefix) { requiredPrefixes = ArrayUtil.append(requiredPrefixes, prefix); return this; } + @JsonIgnore @Getter(lazy=true) private final Set requiredPrefixSet = new HashSet<>(Arrays.asList(getRequiredPrefixes())); + + @Getter @Setter private boolean includeRootFiles = true; + @Getter @Setter private File counterFile = null; + public boolean hasCounterFile () { return counterFile != null; } + + public boolean required(String name) { + return getRequiredClassSet().contains(JarTrimmer.toClassName(name)) + || requiredByPrefix(name) + || (includeRootFiles && !name.contains("/")); + } + + private boolean requiredByPrefix(String name) { + for (String prefix : getRequiredPrefixSet()) if (name.startsWith(prefix)) return true; + return false; + } + + public JarTrimmerConfig requireClasses(File file) throws IOException { + requiredClasses = ArrayUtil.append(requiredClasses, FileUtils.readLines(file, UTF8cs).toArray(new String[0])); + return this; + } +} diff --git a/src/main/java/org/cobbzilla/util/io/PathListFileResolver.java b/src/main/java/org/cobbzilla/util/io/PathListFileResolver.java new file mode 100644 index 0000000..bbb0c1f --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/PathListFileResolver.java @@ -0,0 +1,22 @@ +package org.cobbzilla.util.io; + +import org.cobbzilla.util.collection.StringSetSource; + +import java.io.File; +import java.util.Collection; + +public class PathListFileResolver extends StringSetSource implements FileResolver { + + public PathListFileResolver(Collection paths) { addValues(paths); } + + @Override public File resolve(String path) { + for (String val : getValues()) { + if (!val.endsWith(File.separator)) val += File.separator; + if (path.startsWith(File.separator)) path = path.substring(File.separator.length()); + final File f = new File(val + path); + if (f.exists() && f.canRead()) return f; + } + return null; + } + +} diff --git a/src/main/java/org/cobbzilla/util/io/RegularFileFilter.java b/src/main/java/org/cobbzilla/util/io/RegularFileFilter.java new file mode 100644 index 0000000..628af6b --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/RegularFileFilter.java @@ -0,0 +1,12 @@ +package org.cobbzilla.util.io; + +import java.io.File; +import java.io.FileFilter; + +public class RegularFileFilter implements FileFilter { + + public static final RegularFileFilter instance = new RegularFileFilter(); + + @Override public boolean accept(File pathname) { return pathname.isFile(); } + +} diff --git a/src/main/java/org/cobbzilla/util/io/StreamUtil.java b/src/main/java/org/cobbzilla/util/io/StreamUtil.java new file mode 100644 index 0000000..ec875ef --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/StreamUtil.java @@ -0,0 +1,225 @@ +package org.cobbzilla.util.io; + +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.cobbzilla.util.string.StringUtil; + +import java.io.*; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.daemon.ZillaRuntime.stdin; +import static org.cobbzilla.util.io.FileUtil.basename; +import static org.cobbzilla.util.io.FileUtil.extensionOrName; +import static org.cobbzilla.util.io.FileUtil.getDefaultTempDir; + +@Slf4j +public class StreamUtil { + + public static final String SUFFIX = ".tmp"; + public static final String PREFIX = "stream2file"; + public static final String CLASSPATH_PROTOCOL = "classpath://"; + + public static File stream2temp (String path) { return stream2file(loadResourceAsStream(path), true, path); } + public static File stream2temp (InputStream in) { return stream2file(in, true); } + + public static File stream2file (InputStream in) { return stream2file(in, false); } + public static File stream2file (InputStream in, boolean deleteOnExit) { return stream2file(in, deleteOnExit, SUFFIX); } + + public static File stream2file (InputStream in, boolean deleteOnExit, String pathOrSuffix) { + try { + return stream2file(in, mktemp(deleteOnExit, pathOrSuffix)); + } catch (IOException e) { + return die("stream2file: "+e, e); + } + } + + public static File mktemp(boolean deleteOnExit, String pathOrSuffix) throws IOException { + final String basename = empty(pathOrSuffix) ? "" : basename(pathOrSuffix); + final File file = File.createTempFile( + !basename.contains(".") || basename.length() < 7 ? basename.replace('.', '_')+"_"+PREFIX : basename.split("\\.")[0], + empty(pathOrSuffix) ? SUFFIX : extensionOrName(pathOrSuffix), + getDefaultTempDir()); + if (deleteOnExit) file.deleteOnExit(); + return file; + } + + public static File stream2file(InputStream in, File file) throws IOException { + try (OutputStream out = new FileOutputStream(file)) { + IOUtils.copy(in, out); + } + return file; + } + + public static ByteArrayInputStream toStream(String s) throws UnsupportedEncodingException { + return new ByteArrayInputStream(s.getBytes(StringUtil.UTF8)); + } + + public static String toString(InputStream in) throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOUtils.copy(in, out); + return out.toString(); + } + + public static String toStringOrDie(InputStream in) { + try { return toString(in); } catch (Exception e) { return die("toStringOrDie: "+e, e); } + } + + public static InputStream loadResourceAsStream(String path) { + return loadResourceAsStream(path, StreamUtil.class); + } + + public static InputStream loadResourceAsStream(String path, Class clazz) { + InputStream in = clazz.getClassLoader().getResourceAsStream(path); + if (in == null) throw new IllegalArgumentException("Resource not found: " + path); + return in; + } + + public static File loadResourceAsFile (String path) throws IOException { + final File tmp = File.createTempFile("resource", extensionOrName(path), getDefaultTempDir()); + return loadResourceAsFile(path, StreamUtil.class, tmp); + } + + public static File loadResourceAsFile (String path, Class clazz) throws IOException { + final File tmp = File.createTempFile("resource", ".tmp", getDefaultTempDir()); + return loadResourceAsFile(path, clazz, tmp); + } + + public static File loadResourceAsFile(String path, File file) throws IOException { + return loadResourceAsFile(path, StreamUtil.class, file); + } + + public static File loadResourceAsFile(String path, Class clazz, File file) throws IOException { + if (file.isDirectory()) file = new File(file, new File(path).getName()); + @Cleanup final FileOutputStream out = new FileOutputStream(file); + IOUtils.copy(loadResourceAsStream(path, clazz), out); + return file; + } + + public static String stream2string(String path) { return loadResourceAsStringOrDie(path); } + + public static String stream2string(String path, String defaultValue) { + try { + return loadResourceAsStringOrDie(path); + } catch (Exception e) { + log.info("stream2string: path not found ("+path+": "+e+"), returning defaultValue"); + return defaultValue; + } + } + + public static byte[] stream2bytes(String path) { return loadResourceAsBytesOrDie(path); } + + public static byte[] stream2bytes(String path, byte[] defaultValue) { + try { + return loadResourceAsBytesOrDie(path); + } catch (Exception e) { + log.info("stream2bytes: path not found ("+path+": "+e+"), returning defaultValue"); + return defaultValue; + } + } + + public static byte[] loadResourceAsBytesOrDie(String path) { + try { + @Cleanup final InputStream in = loadResourceAsStream(path); + if (in == null) return die("stream2bytes: not found: "+path); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOUtils.copyLarge(in, out); + return out.toByteArray(); + } catch (Exception e) { + return die("loadResourceAsBytesOrDie: error copying bytes: "+e, e); + } + } + + public static String loadResourceAsStringOrDie(String path) { + try { + return loadResourceAsString(path, StreamUtil.class); + } catch (Exception e) { + throw new IllegalArgumentException("cannot load resource: "+path+": "+e, e); + } + } + + public static String loadResourceAsString(String path) throws IOException { + return loadResourceAsString(path, StreamUtil.class); + } + + public static String loadResourceAsString(String path, Class clazz) throws IOException { + @Cleanup final InputStream in = loadResourceAsStream(path, clazz); + @Cleanup final ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOUtils.copy(in, out); + return out.toString(StringUtil.UTF8); + } + + public static Reader loadResourceAsReader(String resourcePath, Class clazz) throws IOException { + return new InputStreamReader(loadResourceAsStream(resourcePath, clazz)); + } + + public static final int DEFAULT_BUFFER_SIZE = 32 * 1024; + + public static long copyLarge(InputStream input, OutputStream output) throws IOException { + return copyLarge(input, output, DEFAULT_BUFFER_SIZE); + } + + public static long copyLarge(InputStream input, OutputStream output, int bufferSize) throws IOException { + byte[] buffer = new byte[bufferSize]; + long count = 0; + int n = 0; + while (-1 != (n = input.read(buffer))) { + output.write(buffer, 0, n); + count += n; + } + return count; + } + + /** + * Copy the first n bytes from input to output + * @return the number of bytes actually copied (might be less than n if EOF was reached) + */ + public static long copyNbytes(InputStream input, OutputStream output, long n) throws IOException { + byte[] buffer = new byte[(n > DEFAULT_BUFFER_SIZE) ? DEFAULT_BUFFER_SIZE : (int) n]; + long copied = 0; + int read = 0; + while (copied < n && -1 != (read = input.read(buffer, 0, (int) (n - copied > buffer.length ? buffer.length : n - copied)))) { + output.write(buffer, 0, read); + copied += read; + } + return copied; + } + + // incredibly inefficient. do not use frequently. meant for command-line tools that call it no more than a few times + public static String readLineFromStdin() { + final String line; + final BufferedReader r = stdin(); + try { line = r.readLine(); } catch (Exception e) { + return die("Error reading from stdin: " + e); + } + return line == null ? null : line.trim(); + } + + public static String readLineFromStdin(String prompt) { + System.out.print(prompt); + return readLineFromStdin(); + } + + public static String fromClasspathOrFilesystem(String path) { + try { + return stream2string(path); + } catch (Exception e) { + try { + return FileUtil.toStringOrDie(path); + } catch (Exception e2) { + return die("path not found: "+path); + } + } + } + + public static String fromClasspathOrString(String path) { + final boolean isClasspath = path.startsWith(CLASSPATH_PROTOCOL); + if (isClasspath) { + path = path.substring(CLASSPATH_PROTOCOL.length()); + return stream2string(path); + } + return path; + } + +} diff --git a/src/main/java/org/cobbzilla/util/io/Tarball.java b/src/main/java/org/cobbzilla/util/io/Tarball.java new file mode 100644 index 0000000..b1dcc27 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/Tarball.java @@ -0,0 +1,150 @@ +package org.cobbzilla.util.io; + +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.compress.archivers.ArchiveException; +import org.apache.commons.compress.archivers.ArchiveStreamFactory; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.CompressorInputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.exec.CommandLine; +import org.apache.commons.io.FileUtils; +import org.cobbzilla.util.system.Command; + +import java.io.*; + +import static java.io.File.createTempFile; +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.system.CommandShell.*; + +@Slf4j +public class Tarball { + + /** + * @param tarball the tarball to unroll. Can be .tar.gz or .tar.bz2 + * @return a File representing the temp directory where the tarball was unrolled + */ + public static TempDir unroll (File tarball) throws Exception { + TempDir tempDirectory = new TempDir(); + try { + unroll(tarball, tempDirectory); + return tempDirectory; + + } catch (Exception e) { + FileUtils.deleteDirectory(tempDirectory); + throw e; + } + } + + public static void unroll(File tarball, File dir) throws IOException, ArchiveException { + + final String path = abs(tarball); + final FileInputStream fileIn = new FileInputStream(tarball); + final CompressorInputStream zipIn; + + if (path.toLowerCase().endsWith(".gz") || path.toLowerCase().endsWith(".tgz")) { + zipIn = new GzipCompressorInputStream(fileIn); + + } else if (path.toLowerCase().endsWith(".bz2")) { + zipIn = new BZip2CompressorInputStream(fileIn); + + } else { + log.warn("tarball (" + path + ") was not .tar.gz, .tgz, or .tar.bz2, assuming .tar.gz"); + zipIn = new GzipCompressorInputStream(fileIn); + } + + @Cleanup final TarArchiveInputStream tarIn + = (TarArchiveInputStream) new ArchiveStreamFactory() + .createArchiveInputStream("tar", zipIn); + + TarArchiveEntry entry; + while ((entry = tarIn.getNextTarEntry()) != null) { + String name = entry.getName(); + if (name.startsWith("./")) name = name.substring(2); + if (name.startsWith("/")) name = name.substring(1); // "root"-based files just go into current dir + if (name.endsWith("/")) { + final String subdirName = name.substring(0, name.length() - 1); + final File subdir = new File(dir, subdirName); + if (!subdir.exists() && !subdir.mkdirs()) { + die("Error creating directory: " + abs(subdir)); + } + continue; + } + + // when "./" gets squashed to "", we skip the entry + if (name.trim().length() == 0) continue; + + final File file = new File(dir, name); + try (OutputStream out = new FileOutputStream(file)) { + if (StreamUtil.copyNbytes(tarIn, out, entry.getSize()) != entry.getSize()) { + die("Expected to copy "+entry.getSize()+ " bytes for "+entry.getName()+" in tarball "+ path); + } + } + chmod(file, Integer.toOctalString(entry.getMode())); + } + } + + /** + * Roll a gzipped tarball. The tarball will be created from within the directory to be tarred (paths will be relative to .) + * @param dir The directory to tar + * @return The created tarball (will be a temp file) + */ + public static File roll (File dir) throws IOException { + return roll(createTempFile("temp-tarball-", ".tar.gz"), dir, dir); + } + + /** + * Roll a gzipped tarball. The tarball will be created from within the directory to be tarred (paths will be relative to .) + * @param tarball The path to the tarball to create + * @param dir The directory to tar + * @return The created tarball + */ + public static File roll (File tarball, File dir) throws IOException { + return roll(tarball, dir, dir); + } + + /** + * Roll a gzipped tarball. The tarball will be created from "cwd", which must above the directory to be tarred. + * @param tarball The path to the tarball to create + * @param dir The directory to tar + * @param cwd A directory that is somewhere above dir in the filesystem hierarchy + * @return The created tarball + */ + public static File roll (File tarball, File dir, File cwd) throws IOException { + + if (cwd == null) cwd = dir; + final String dirAbsPath = abs(dir); + final String cwdAbsPath = abs(cwd); + + final String dirPath; + if (dirAbsPath.equals(cwdAbsPath)) { + dirPath = "."; + + } else if (dirAbsPath.startsWith(cwdAbsPath)) { + dirPath = cwdAbsPath.substring(dirAbsPath.length()); + + } else { + return die("tarball dir is not within cwd"); + } + + final CommandLine command = new CommandLine("tar") + .addArgument("czf") + .addArgument(tarball.getAbsolutePath()) + .addArgument(dirPath); + + okResult(exec(new Command(command).setDir(cwd))); + + return tarball; + } + + public static boolean isTarball(File file) { return isTarball(file.getName().toLowerCase()); } + + public static boolean isTarball(String fileName) { + return fileName.endsWith(".tar.gz") + || fileName.endsWith(".tar.bz2") + || fileName.endsWith(".tgz"); + } +} diff --git a/src/main/java/org/cobbzilla/util/io/TempDir.java b/src/main/java/org/cobbzilla/util/io/TempDir.java new file mode 100644 index 0000000..b32c10d --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/TempDir.java @@ -0,0 +1,124 @@ +package org.cobbzilla.util.io; + +import com.google.common.io.Files; +import lombok.AllArgsConstructor; +import lombok.Delegate; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.util.SortedSet; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.TimeUnit; + +import static org.cobbzilla.util.daemon.ZillaRuntime.*; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.io.FileUtil.getDefaultTempDir; +import static org.cobbzilla.util.system.CommandShell.chmod; +import static org.cobbzilla.util.system.Sleep.sleep; + +/** + * A directory that implements Closeable. Use lombok @Cleanup to nuke it when it goes out of scope. + */ +@Slf4j +public class TempDir extends File implements Closeable { + + @AllArgsConstructor + private static class FileKillOrder implements Comparable { + @Getter @Setter private File file; + @Getter @Setter private long killTime; + @Override public int compareTo(FileKillOrder k) { + if (killTime > k.getKillTime()) return 1; + if (killTime == k.getKillTime()) return 0; + return -1; + } + public boolean shouldKill() { return killTime != QT_NO_DELETE && now() > killTime; } + } + + private static class QuickTempReaper implements Runnable { + private final SortedSet temps = new ConcurrentSkipListSet<>(); + public File add (File t) { return add(t, now() + TimeUnit.MINUTES.toMillis(5)); } + public File add (File t, long killTime) { + synchronized (temps) { + temps.add(new FileKillOrder(t, killTime)); + return t; + } + } + @Override public void run() { + while (true) { + sleep(10_000); + synchronized (temps) { + while (!temps.isEmpty() && temps.first().shouldKill()) { + if (!temps.first().getFile().delete()) { + log.warn("QuickTempReaper.run: couldn't delete " + abs(temps.first().getFile())); + } + temps.remove(temps.first()); + } + } + } + } + public QuickTempReaper start () { + daemon(this); + return this; + } + } + + private static QuickTempReaper qtReaper = new QuickTempReaper().start(); + + public static final long QT_NO_DELETE = -1L; + + public static File quickTemp() { return quickTemp(TimeUnit.MINUTES.toMillis(5)); } + + public static File quickTemp(final long killAfter) { + try { + final File temp = temp(); + if (killAfter > 0) { + long killTime = killAfter + now(); + killAfter(temp, killTime); + } + return temp; + } catch (IOException e) { + return die("quickTemp: cannot create temp file: " + e, e); + } + } + + public static void killAfter(File temp, long killTime) { killAt(temp, killTime + now()); } + + private static void killAt(File temp, long t) { qtReaper.add(temp, t); } + + private static File temp() throws IOException { + return File.createTempFile("quickTemp-", ".tmp", getDefaultTempDir()); + } + + private interface TempDirOverrides { boolean delete(); } + + @Delegate(excludes=TempDirOverrides.class) + private File file; + + public TempDir () { this("700"); } + + public TempDir (String chmod) { + super(abs(Files.createTempDir())); + file = new File(super.getPath()); + chmod(file, chmod); + } + + public void killAfter (long t) { + killAfter(this, t); + } + + @Override public void close() throws IOException { + if (!delete()) log.warn("close: error deleting TempDir: "+abs(file)); + } + + /** + * Override to call 'delete', delete the entire directory. + * @return true if the delete was successful. + */ + @Override public boolean delete() { return FileUtils.deleteQuietly(file); } + +} diff --git a/src/main/java/org/cobbzilla/util/io/UniqueFileFsWalker.java b/src/main/java/org/cobbzilla/util/io/UniqueFileFsWalker.java new file mode 100644 index 0000000..14c042b --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/UniqueFileFsWalker.java @@ -0,0 +1,27 @@ +package org.cobbzilla.util.io; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.security.ShaUtil.sha256_file; + +@Slf4j +public class UniqueFileFsWalker implements FilesystemVisitor { + + @Getter private Map> hash; + + public UniqueFileFsWalker (int size) { hash = new ConcurrentHashMap<>(size); } + + @Override public void visit(File file) { + final String path = abs(file); + log.debug(path); + hash.computeIfAbsent(sha256_file(file), k -> new HashSet<>()).add(path); + } +} diff --git a/src/main/java/org/cobbzilla/util/io/handlers/ConfigurableStreamHandlerFactory.java b/src/main/java/org/cobbzilla/util/io/handlers/ConfigurableStreamHandlerFactory.java new file mode 100644 index 0000000..5a3c625 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/handlers/ConfigurableStreamHandlerFactory.java @@ -0,0 +1,32 @@ +package org.cobbzilla.util.io.handlers; + +import org.cobbzilla.util.io.handlers.classpath.Handler; + +import java.net.URLStreamHandler; +import java.net.URLStreamHandlerFactory; +import java.util.HashMap; +import java.util.Map; + +// from https://stackoverflow.com/a/1769454/1251543 +public class ConfigurableStreamHandlerFactory implements URLStreamHandlerFactory { + + public static final ConfigurableStreamHandlerFactory INSTANCE = new ConfigurableStreamHandlerFactory(); + + private final Map protocolHandlers = new HashMap<>(); + + public ConfigurableStreamHandlerFactory() { addAll(); } + + public ConfigurableStreamHandlerFactory(String protocol, URLStreamHandler urlHandler) { + addHandler(protocol, urlHandler); + } + + public ConfigurableStreamHandlerFactory addAll () { + addHandler("classpath", new Handler()); + return this; + } + + public void addHandler(String protocol, URLStreamHandler urlHandler) { protocolHandlers.put(protocol, urlHandler); } + + public URLStreamHandler createURLStreamHandler(String protocol) { return protocolHandlers.get(protocol); } + +} \ No newline at end of file diff --git a/src/main/java/org/cobbzilla/util/io/handlers/classpath/Handler.java b/src/main/java/org/cobbzilla/util/io/handlers/classpath/Handler.java new file mode 100644 index 0000000..51e2a85 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/handlers/classpath/Handler.java @@ -0,0 +1,23 @@ +package org.cobbzilla.util.io.handlers.classpath; + +import java.io.IOException; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLStreamHandler; + +// from https://stackoverflow.com/a/1769454/1251543 +public class Handler extends URLStreamHandler { + + /** The classloader to find resources from. */ + private final ClassLoader classLoader; + + public Handler() { this.classLoader = getClass().getClassLoader(); } + + public Handler(ClassLoader classLoader) { this.classLoader = classLoader; } + + @Override protected URLConnection openConnection(URL u) throws IOException { + final URL resource = classLoader.getResource(u.getPath()); + return resource == null ? null : resource.openConnection(); + } + +} \ No newline at end of file diff --git a/src/main/java/org/cobbzilla/util/io/main/FilesystemWatcherMain.java b/src/main/java/org/cobbzilla/util/io/main/FilesystemWatcherMain.java new file mode 100644 index 0000000..cb7c7de --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/main/FilesystemWatcherMain.java @@ -0,0 +1,53 @@ +package org.cobbzilla.util.io.main; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.exec.CommandLine; +import org.cobbzilla.util.io.DamperedCompositeBufferedFilesystemWatcher; +import org.cobbzilla.util.main.BaseMain; +import org.cobbzilla.util.system.CommandShell; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import java.nio.file.WatchEvent; +import java.util.List; + +import static org.cobbzilla.util.daemon.ZillaRuntime.errorString; +import static org.cobbzilla.util.daemon.ZillaRuntime.now; + +@Slf4j +public class FilesystemWatcherMain extends BaseMain { + + public static final DateTimeFormatter DFORMAT = DateTimeFormat.forPattern("yyyy-MMM-dd HH:mm:ss"); + + public static void main (String[] args) { main(FilesystemWatcherMain.class, args); } + + protected void run() throws Exception { + + final FilesystemWatcherMainOptions options = getOptions(); + + final DamperedCompositeBufferedFilesystemWatcher watcher = new DamperedCompositeBufferedFilesystemWatcher + (options.getTimeout(), options.getMaxEvents(), options.getWatchPaths(), options.getDamperMillis()) { + @Override public void uber_fire(List> events) { + try { + if (options.hasCommand()) { + CommandShell.exec(new CommandLine(options.getCommand())); + } else { + final String msg = status() + " uber_fire ("+events.size()+" events) at " + DFORMAT.print(now()); + log.info(msg); + System.out.println(msg); + } + } catch (Exception e) { + final String msg = status() + " Error running command (" + options.getCommand() + "): " + + errorString(e); + log.error(msg, e); + System.err.println(msg); + } + } + }; + + synchronized (watcher) { + watcher.wait(); + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/io/main/FilesystemWatcherMainOptions.java b/src/main/java/org/cobbzilla/util/io/main/FilesystemWatcherMainOptions.java new file mode 100644 index 0000000..5a6d227 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/main/FilesystemWatcherMainOptions.java @@ -0,0 +1,51 @@ +package org.cobbzilla.util.io.main; + +import lombok.Getter; +import lombok.Setter; +import org.cobbzilla.util.collection.SingletonSet; +import org.cobbzilla.util.main.BaseMainOptions; +import org.kohsuke.args4j.Argument; +import org.kohsuke.args4j.Option; + +import java.util.Collection; +import java.util.List; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +public class FilesystemWatcherMainOptions extends BaseMainOptions { + + public static final String USAGE_COMMAND = "Command to run when something changes, must be executable. Default is to print the changes detected."; + public static final String OPT_COMMAND = "-c"; + public static final String LONGOPT_COMMAND = "--command"; + @Option(name=OPT_COMMAND, aliases=LONGOPT_COMMAND, usage=USAGE_COMMAND, required=false) + @Getter @Setter private String command = null; + public boolean hasCommand () { return !empty(command); } + + public static final int DEFAULT_TIMEOUT = 600; + public static final String USAGE_TIMEOUT = "Command will be run after this timeout (in seconds), regardless of any changes. Default is "+DEFAULT_TIMEOUT+" seconds ("+DEFAULT_TIMEOUT/60+" minutes)."; + public static final String OPT_TIMEOUT = "-t"; + public static final String LONGOPT_TIMEOUT = "--timeout"; + @Option(name=OPT_TIMEOUT, aliases=LONGOPT_TIMEOUT, usage=USAGE_TIMEOUT, required=false) + @Getter @Setter private int timeout = DEFAULT_TIMEOUT; + + public static final int DEFAULT_MAXEVENTS = 100; + public static final String USAGE_MAXEVENTS = "Command will be run after this many events have occurred. Default is " + DEFAULT_MAXEVENTS; + public static final String OPT_MAXEVENTS = "-m"; + public static final String LONGOPT_MAXEVENTS = "--max-events"; + @Option(name=OPT_MAXEVENTS, aliases=LONGOPT_MAXEVENTS, usage=USAGE_MAXEVENTS, required=false) + @Getter @Setter private int maxEvents = DEFAULT_MAXEVENTS; + + public static final String USAGE_DAMPER = "Command will never be run until there have been no events for this many seconds. Default is 0 (disabled). Takes precedence over "+OPT_TIMEOUT+"/"+LONGOPT_TIMEOUT; + public static final String OPT_DAMPER = "-d"; + public static final String LONGOPT_DAMPER = "--damper"; + @Option(name=OPT_DAMPER, aliases=LONGOPT_DAMPER, usage=USAGE_DAMPER, required=false) + @Getter @Setter private int damper = 0; + public long getDamperMillis () { return damper * 1000; } + + public static final String USAGE_PATHS = "Paths to watch for changes. Default is the current directory"; + @Argument(usage=USAGE_PATHS) + @Getter @Setter private List paths = null; + + public boolean hasPaths () { return !empty(paths); } + public Collection getWatchPaths() { return hasPaths() ? paths : new SingletonSet(System.getProperty("user.dir")); } +} diff --git a/src/main/java/org/cobbzilla/util/io/main/JarTrimmerMain.java b/src/main/java/org/cobbzilla/util/io/main/JarTrimmerMain.java new file mode 100644 index 0000000..28e6617 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/main/JarTrimmerMain.java @@ -0,0 +1,16 @@ +package org.cobbzilla.util.io.main; + +import org.cobbzilla.util.io.JarTrimmer; +import org.cobbzilla.util.main.BaseMain; + +public class JarTrimmerMain extends BaseMain { + + public static void main (String[] args) { main(JarTrimmerMain.class, args); } + + @Override protected void run() throws Exception { + final JarTrimmerOptions opts = getOptions(); + final JarTrimmer trimmer = new JarTrimmer(); + trimmer.trim(opts.getConfig()); + } + +} diff --git a/src/main/java/org/cobbzilla/util/io/main/JarTrimmerOptions.java b/src/main/java/org/cobbzilla/util/io/main/JarTrimmerOptions.java new file mode 100644 index 0000000..d77caa2 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/main/JarTrimmerOptions.java @@ -0,0 +1,24 @@ +package org.cobbzilla.util.io.main; + +import lombok.Getter; +import lombok.Setter; +import org.cobbzilla.util.io.FileUtil; +import org.cobbzilla.util.io.JarTrimmerConfig; +import org.cobbzilla.util.main.BaseMainOptions; +import org.kohsuke.args4j.Option; + +import java.io.File; + +import static org.cobbzilla.util.json.JsonUtil.json; + +public class JarTrimmerOptions extends BaseMainOptions { + + public static final String USAGE_CONFIG = "JSON configuration file to drive JarTrimmer"; + public static final String OPT_CONFIG = "-c"; + public static final String LONGOPT_CONFIG= "--config"; + @Option(name=OPT_CONFIG, aliases=LONGOPT_CONFIG, usage=USAGE_CONFIG) + @Getter @Setter private File configFile; + + public JarTrimmerConfig getConfig() { return json(FileUtil.toStringOrDie(getConfigFile()), JarTrimmerConfig.class); } + +} diff --git a/src/main/java/org/cobbzilla/util/io/main/RegexFilterMain.java b/src/main/java/org/cobbzilla/util/io/main/RegexFilterMain.java new file mode 100644 index 0000000..d49fc22 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/main/RegexFilterMain.java @@ -0,0 +1,30 @@ +package org.cobbzilla.util.io.main; + +import lombok.Cleanup; +import org.apache.commons.io.IOUtils; +import org.cobbzilla.util.io.regex.RegexFilterReader; +import org.cobbzilla.util.io.regex.RegexStreamFilter; +import org.cobbzilla.util.main.BaseMain; + +import java.io.OutputStreamWriter; + +public class RegexFilterMain extends BaseMain { + + public static void main (String[] args) { main(RegexFilterMain.class, args); } + + @Override protected void run() throws Exception { + + final RegexFilterOptions options = getOptions(); + + final RegexStreamFilter filter = getDriver(options); + filter.configure(options.getConfig()); + + @Cleanup final RegexFilterReader reader = new RegexFilterReader(System.in, options.getBufferSize(), filter); + @Cleanup final OutputStreamWriter out = new OutputStreamWriter(System.out); + + IOUtils.copyLarge(reader, out); + } + + protected RegexStreamFilter getDriver(RegexFilterOptions options) { return options.getDriver(); } + +} diff --git a/src/main/java/org/cobbzilla/util/io/main/RegexFilterOptions.java b/src/main/java/org/cobbzilla/util/io/main/RegexFilterOptions.java new file mode 100644 index 0000000..6044109 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/main/RegexFilterOptions.java @@ -0,0 +1,52 @@ +package org.cobbzilla.util.io.main; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Getter; +import lombok.Setter; +import org.cobbzilla.util.io.FileUtil; +import org.cobbzilla.util.io.regex.RegexStreamFilter; +import org.cobbzilla.util.main.BaseMainOptions; +import org.cobbzilla.util.system.Bytes; +import org.kohsuke.args4j.Option; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.reflect.ReflectionUtil.instantiate; + +public class RegexFilterOptions extends BaseMainOptions { + + public static final String USAGE_BUFSIZ = "Buffer size (default 16K)"; + public static final String OPT_BUFSIZ = "-b"; + public static final String LONGOPT_BUFSIZ= "--buffer-size"; + @Option(name=OPT_BUFSIZ, aliases=LONGOPT_BUFSIZ, usage=USAGE_BUFSIZ) + @Getter @Setter private int bufferSize = (int) (16 * Bytes.KB); + + public static final String USAGE_DRIVER = "Driver class"; + public static final String OPT_DRIVER = "-d"; + public static final String LONGOPT_DRIVER= "--driver"; + @Option(name=OPT_DRIVER, aliases=LONGOPT_DRIVER, usage=USAGE_DRIVER, required=true) + @Getter @Setter private String driverClass; + + public RegexStreamFilter getDriver() { + try { + return instantiate(getDriverClass()); + } catch (Exception e) { + return die("getDriver: "+e, e); + } + } + + public static final String USAGE_CONFIG = "Configuration. If first char is @ then this is a file path. Otherwise it is literal JSON"; + public static final String OPT_CONFIG = "-c"; + public static final String LONGOPT_CONFIG= "--config"; + @Option(name=OPT_CONFIG, aliases=LONGOPT_CONFIG, usage=USAGE_CONFIG) + @Getter @Setter private String configJson; + + public JsonNode getConfig () { + if (configJson == null) return null; + if (configJson.startsWith("@")) { + return json(FileUtil.toStringOrDie(configJson.substring(1)), JsonNode.class); + } + return json(configJson, JsonNode.class); + } + +} diff --git a/src/main/java/org/cobbzilla/util/io/main/UniqueFileWalkerMain.java b/src/main/java/org/cobbzilla/util/io/main/UniqueFileWalkerMain.java new file mode 100644 index 0000000..dd0f2ba --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/main/UniqueFileWalkerMain.java @@ -0,0 +1,58 @@ +package org.cobbzilla.util.io.main; + +import lombok.Cleanup; +import org.apache.commons.io.output.TeeOutputStream; +import org.cobbzilla.util.daemon.AwaitResult; +import org.cobbzilla.util.io.FilesystemWalker; +import org.cobbzilla.util.io.UniqueFileFsWalker; +import org.cobbzilla.util.main.BaseMain; +import org.cobbzilla.util.string.StringUtil; + +import java.io.FileOutputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.Set; + +import static java.util.stream.Collectors.toList; + +public class UniqueFileWalkerMain extends BaseMain { + + public static void main (String[] args) { main(UniqueFileWalkerMain.class, args); } + + @Override protected void run() throws Exception { + final UniqueFileWalkerOptions options = getOptions(); + + final UniqueFileFsWalker visitor = new UniqueFileFsWalker(options.getSize()); + final AwaitResult result = new FilesystemWalker() + .setSize(options.getSize()) + .setThreads(options.getThreads()) + .withDirs(options.getDirs()) + .withTimeoutDuration(options.getTimeoutDuration()) + .withVisitor(visitor) + .walk(); + + if (!result.allSucceeded()) { + if (result.numFails() > 0) { + out(">>>>> "+result.getFailures().values().size()+" failures:"); + out(StringUtil.toString(result.getFailures().values(), "\n-----")); + } + if (result.numTimeouts() > 0) { + out(">>>>> "+result.getTimeouts().size()+" timeouts"); + } + } + int i=1; + OutputStream out; + if (options.hasOutfile()) { + out = new TeeOutputStream(new FileOutputStream(options.getOutfile()), System.out); + } else { + out = System.out; + } + @Cleanup final Writer w = new OutputStreamWriter(out); + for (Set dup : visitor.getHash().values().stream().filter(v -> v.size() > 1).collect(toList())) { + w.write("\n----- dup#" + (i++) + ": \n"); + w.write(StringUtil.toString(dup, "\n")); + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/io/main/UniqueFileWalkerOptions.java b/src/main/java/org/cobbzilla/util/io/main/UniqueFileWalkerOptions.java new file mode 100644 index 0000000..f909307 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/main/UniqueFileWalkerOptions.java @@ -0,0 +1,43 @@ +package org.cobbzilla.util.io.main; + +import lombok.Getter; +import lombok.Setter; +import org.cobbzilla.util.main.BaseMainOptions; +import org.kohsuke.args4j.Option; + +import java.io.File; + +public class UniqueFileWalkerOptions extends BaseMainOptions { + + public static final String USAGE_DIR = "Add a directory to the search"; + public static final String OPT_DIR = "-d"; + public static final String LONGOPT_DIR= "--dir"; + @Option(name=OPT_DIR, aliases=LONGOPT_DIR, usage=USAGE_DIR, required=true) + @Getter @Setter private File[] dirs; + + public static final String USAGE_TIMEOUT = "Timeout duration. For example 10m for ten minutes. use h for hours, d for days."; + public static final String OPT_TIMEOUT = "-t"; + public static final String LONGOPT_TIMEOUT= "--timeout"; + @Option(name=OPT_TIMEOUT, aliases=LONGOPT_TIMEOUT, usage=USAGE_TIMEOUT) + @Getter @Setter private String timeoutDuration = "1d"; + + public static final String USAGE_SIZE = "Rough guess to number of files to visit."; + public static final String OPT_SIZE = "-s"; + public static final String LONGOPT_SIZE= "--size"; + @Option(name=OPT_SIZE, aliases=LONGOPT_SIZE, usage=USAGE_SIZE) + @Getter @Setter private int size = 1_000_000; + + public static final String USAGE_THREADS = "Degree of parallelism"; + public static final String OPT_THREADS = "-p"; + public static final String LONGOPT_THREADS= "--parallel"; + @Option(name=OPT_THREADS, aliases=LONGOPT_THREADS, usage=USAGE_THREADS) + @Getter @Setter private int threads = 5; + + public static final String USAGE_OUTFILE = "Output file"; + public static final String OPT_OUTFILE = "-o"; + public static final String LONGOPT_OUTFILE= "--outfile"; + @Option(name=OPT_OUTFILE, aliases=LONGOPT_OUTFILE, usage=USAGE_OUTFILE) + @Getter @Setter private File outfile; + public boolean hasOutfile () { return outfile != null; } + +} diff --git a/src/main/java/org/cobbzilla/util/io/main/UnrollMain.java b/src/main/java/org/cobbzilla/util/io/main/UnrollMain.java new file mode 100644 index 0000000..c64fb5b --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/main/UnrollMain.java @@ -0,0 +1,16 @@ +package org.cobbzilla.util.io.main; + +import org.cobbzilla.util.main.BaseMain; + +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.io.Decompressors.unroll; + +public class UnrollMain extends BaseMain { + + public static void main (String[] args) { main(UnrollMain.class, args); } + + @Override protected void run() throws Exception { + out(abs(unroll(getOptions().getFile()))); + } + +} diff --git a/src/main/java/org/cobbzilla/util/io/main/UnrollOptions.java b/src/main/java/org/cobbzilla/util/io/main/UnrollOptions.java new file mode 100644 index 0000000..7a7422f --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/main/UnrollOptions.java @@ -0,0 +1,18 @@ +package org.cobbzilla.util.io.main; + +import lombok.Getter; +import lombok.Setter; +import org.cobbzilla.util.main.BaseMainOptions; +import org.kohsuke.args4j.Option; + +import java.io.File; + +public class UnrollOptions extends BaseMainOptions { + + public static final String USAGE_FILE = "File to unroll"; + public static final String OPT_FILE = "-f"; + public static final String LONGOPT_FILE= "--file"; + @Option(name=OPT_FILE, aliases=LONGOPT_FILE, usage=USAGE_FILE, required=true) + @Getter @Setter private File file; + +} diff --git a/src/main/java/org/cobbzilla/util/io/regex/RegexChunk.java b/src/main/java/org/cobbzilla/util/io/regex/RegexChunk.java new file mode 100644 index 0000000..0f8d9c4 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/regex/RegexChunk.java @@ -0,0 +1,28 @@ +package org.cobbzilla.util.io.regex; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.Accessors; + +import java.util.HashMap; +import java.util.Map; + +@NoArgsConstructor @Accessors(chain=true) @ToString +public class RegexChunk { + + @Getter @Setter private RegexChunkType type; + @Getter @Setter private String data; + @Getter @Setter private boolean partial = false; + + @Getter private Map properties; + + public String getProperty (String prop) { return properties == null ? null : properties.get(prop); } + + public void setProperty (String prop, String val) { + if (properties == null) properties = new HashMap<>(); + properties.put(prop, val); + } + +} diff --git a/src/main/java/org/cobbzilla/util/io/regex/RegexChunkConfig.java b/src/main/java/org/cobbzilla/util/io/regex/RegexChunkConfig.java new file mode 100644 index 0000000..f9698aa --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/regex/RegexChunkConfig.java @@ -0,0 +1,24 @@ +package org.cobbzilla.util.io.regex; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; +import lombok.Setter; +import org.cobbzilla.util.collection.NameAndValue; + +import java.util.regex.Pattern; + +public class RegexChunkConfig { + + @Getter @Setter private String chunkStartRegex; + @Getter @Setter private String chunkEndRegex; + public boolean hasChunkEndRegex () { return chunkEndRegex != null && chunkEndRegex.length() > 0; } + + private Pattern initPattern(String regex) { return Pattern.compile(regex); } + + @JsonIgnore @Getter(lazy=true) private final Pattern chunkStartPattern = initPattern(getChunkStartRegex()); + @JsonIgnore @Getter(lazy=true) private final Pattern chunkEndPattern = hasChunkEndRegex() ? initPattern(getChunkEndRegex()) : null; + + @Getter @Setter private NameAndValue[] chunkProperties; + public boolean hasChunkProperties () { return chunkProperties != null && chunkProperties.length > 0; } + +} diff --git a/src/main/java/org/cobbzilla/util/io/regex/RegexChunkStreamer.java b/src/main/java/org/cobbzilla/util/io/regex/RegexChunkStreamer.java new file mode 100644 index 0000000..cd6524b --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/regex/RegexChunkStreamer.java @@ -0,0 +1,133 @@ +package org.cobbzilla.util.io.regex; + +import lombok.Getter; +import org.cobbzilla.util.collection.NameAndValue; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class RegexChunkStreamer { + + private RegexChunkConfig config; + private List chunks = new ArrayList<>(); + private int index = 0; + @Getter private int total = 0; + + public boolean hasMoreChunks () { return index < chunks.size(); } + + public RegexChunk nextChunk () { return chunks.get(index++); } + + public RegexChunkStreamer(StringBuilder buffer, RegexChunkConfig config, boolean eof) { + this.config = config; + + final Matcher chunkStartMatcher = config.getChunkStartPattern().matcher(buffer); + + final Pattern chunkEndRegex = config.getChunkEndPattern(); + final Matcher chunkEndMatcher = chunkEndRegex == null ? null : chunkEndRegex.matcher(buffer); + + int start = 0; + while (chunkStartMatcher.find(start)) { + if (chunks.isEmpty() && start != 0 && chunkStartMatcher.start() != 0) { + addChunk(new RegexChunk() + .setType(RegexChunkType.content) + .setData(buffer.substring(0, chunkStartMatcher.start()))); + + } else if (chunkStartMatcher.start() != start) { + // add a content chunk + addChunk(new RegexChunk() + .setType(RegexChunkType.content) + .setData(buffer.substring(start, chunkStartMatcher.start()))); + } + + // figure out where the chunk starts and stops + RegexChunk chunk = null; + if (chunkEndMatcher != null) { + if (chunkEndMatcher.find(chunkStartMatcher.end())) { + chunk = new RegexChunk() + .setType(RegexChunkType.chunk) + .setData(buffer.substring(chunkStartMatcher.start(), chunkEndMatcher.end())); + start = chunkEndMatcher.end(); + + } else { + // chunk end not found, if we are at EOF that is OK, this must be last comment + if (eof) { + chunk = new RegexChunk() + .setType(RegexChunkType.chunk) + .setData(buffer.substring(chunkStartMatcher.start())); + start = buffer.length(); + addChunk(chunk); + + } else { + // this blocked comment must extend to end of buffer + // ask for more data, we have to restart processing + chunk = new RegexChunk() + .setPartial(true) + .setType(RegexChunkType.chunk) + .setData(buffer.substring(chunkStartMatcher.start())); + addChunk(chunk); + return; + } + } + } + if (chunk == null) { + int chunkStart = chunkStartMatcher.start(); + if (chunkStartMatcher.find(chunkStartMatcher.end())) { + // found the next comment, cut up until then + chunk = new RegexChunk() + .setType(RegexChunkType.chunk) + .setData(buffer.substring(chunkStart, chunkStartMatcher.start())); + start = chunkStartMatcher.start(); + } else { + // no more comments! I guess we go from here until the end + if (eof) { + chunk = new RegexChunk() + .setType(RegexChunkType.chunk) + .setData(buffer.substring(chunkStart)); + start = buffer.length(); + } else { + // ask for more data + // this blocked comment must extend to end of buffer + // ask for more data, we have to restart processing + chunk = new RegexChunk() + .setPartial(true) + .setType(RegexChunkType.chunk) + .setData(buffer.substring(chunkStart)); + addChunk(chunk); + return; + } + } + } + addChunk(chunk); + } + if (chunks.isEmpty()) { + // we found nothing, so the whole buffer is the entire thing + addChunk(new RegexChunk() + .setType(RegexChunkType.content) + .setData(buffer.toString())); + } else { + // add any remainder as the footer + addChunk(new RegexChunk() + .setType(RegexChunkType.content) + .setData(buffer.substring(start))); + } + } + + private void addChunk (RegexChunk chunk) { + if (chunk.getType() == RegexChunkType.chunk) { + if (config.hasChunkProperties()) { + for (NameAndValue prop : config.getChunkProperties()) { + final Pattern p = Pattern.compile(prop.getValue(), Pattern.CASE_INSENSITIVE|Pattern.MULTILINE); + final Matcher m = p.matcher(chunk.getData()); + if (m.find()) { + chunk.setProperty(prop.getName(), m.group(1)); + } + } + } + } + chunks.add(chunk); + total += chunk.getData().length(); + } + +} diff --git a/src/main/java/org/cobbzilla/util/io/regex/RegexChunkType.java b/src/main/java/org/cobbzilla/util/io/regex/RegexChunkType.java new file mode 100644 index 0000000..26ddc0a --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/regex/RegexChunkType.java @@ -0,0 +1,7 @@ +package org.cobbzilla.util.io.regex; + +public enum RegexChunkType { + + chunk, content + +} diff --git a/src/main/java/org/cobbzilla/util/io/regex/RegexFilterReader.java b/src/main/java/org/cobbzilla/util/io/regex/RegexFilterReader.java new file mode 100644 index 0000000..c46e542 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/regex/RegexFilterReader.java @@ -0,0 +1,218 @@ +package org.cobbzilla.util.io.regex; + +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ArrayUtils; +import org.cobbzilla.util.system.Bytes; + +import java.io.*; +import java.util.concurrent.atomic.AtomicReference; + +import static org.apache.commons.lang3.ArrayUtils.addAll; +import static org.cobbzilla.util.daemon.ZillaRuntime.die; + +@Slf4j @Accessors(chain=true) +public class RegexFilterReader extends BufferedReader { + + public static final int DEFAULT_BUFFER_SIZE = (int) (64 * Bytes.KB); + + private final int bufsiz; + private RegexStreamFilter filter; + @Getter @Setter private String name; // for debugging/identifying which reader + + public RegexFilterReader(Reader in, RegexStreamFilter filter) { + this(in, DEFAULT_BUFFER_SIZE, filter); + } + + public RegexFilterReader(Reader in, int bufsiz, RegexStreamFilter filter) { + super(in, DEFAULT_BUFFER_SIZE); + this.bufsiz = bufsiz; + this.filter = filter; + } + + public RegexFilterReader(InputStream in, RegexStreamFilter filter) { + this(in, DEFAULT_BUFFER_SIZE, filter); + } + + public RegexFilterReader(InputStream in, int bufsiz, RegexStreamFilter filter) { + super(new InputStreamReader(in), bufsiz); + this.bufsiz = bufsiz; + this.filter = filter; + } + + @Getter(lazy=true) private final FilterResponse filterResponse = initFilterResponse(); + private FilterResponse initFilterResponse() { return new FilterResponse(bufsiz, filter); } + + @Override public int read() throws IOException { + final char[] c = new char[1]; + if (read(c, 0, 1) == -1) return -1; + return c[0]; + } + + @Override public int read(char[] cbuf, int off, int len) throws IOException { + return fillBuffer(len).read(cbuf, off, len); + } + + @Override public String readLine() throws IOException { + return fillBuffer(-1).readLine(); + } + + private FilterResponse fillBuffer(int sz) throws IOException { + return fillBuffer(sz, bufsiz); + } + + private FilterResponse fillBuffer(int sz, int bz) throws IOException { + + // if the filterResponse still has enough data, we don't need to re-fill + final FilterResponse f = getFilterResponse(); + final int readSize = Math.max(sz, bz); + if (f.canRead(readSize)) return f; + + // we need more data + final char[] buffer = new char[readSize]; + int len = 0; + boolean eof = false; + // fill the buffer with the underlying stream + while (len != buffer.length) { + int val = super.read(buffer, len, buffer.length - len); + if (val == -1) { + eof = true; + break; + } + len += val; + } + + // if we read anything, filter the buffer and apply filters + if (len > 0) { + if (f.addData(buffer, len, eof) == 0) { + if (2 * bz < (2 * Bytes.MB)) { + return fillBuffer(2 * bz, 2 * bz); + } else { + // we don't want to run out of memory + return die("fillBuffer: is someone DOSing us?"); + } + } + } + return f; + } + + private static class FilterResponse { + + @Getter @Setter private int bufsiz; + + private int readPos = 0; + private boolean eof = false; + private RegexStreamFilter filter; + + public FilterResponse(int bufsiz, RegexStreamFilter filter) { + this.bufsiz = bufsiz; + this.filter = filter; + } + + private final AtomicReference unprocessed = new AtomicReference<>(); + private final AtomicReference processed = new AtomicReference<>(); + + public int addData(char[] b, int len, boolean eof) { + + this.eof = eof; + + synchronized (unprocessed) { + char[] input = unprocessed.get(); + if (input == null) { + input = new char[len]; + System.arraycopy(b, 0, input, 0, len); + } else { + char[] new_input = new char[input.length + len]; + System.arraycopy(input, 0, new_input, 0, input.length); + System.arraycopy(b, 0, new_input, input.length, len); + input = new_input; + } + unprocessed.set(input); + + // process buffer with filter, it may leave a remainder + final RegexFilterResult result = filter.apply(new StringBuilder().append(input), eof); + + // put unprocessed remainder chars back onto unprocessed array + if (result.remainder > 0) { + final char[] subarray = ArrayUtils.subarray(input, input.length - result.remainder, input.length); + unprocessed.set(subarray); + } else { + unprocessed.set(null); + } + + // if it produced nothing, but gave us a remainder, return zero now, read more data + if (result.buffer.length() == 0 && !eof) return 0; + + // remove processed chars from unprocessed array + synchronized (processed) { + final char[] newChars = result.buffer.toString().toCharArray(); + if (newChars.length > 0) { + if (processed.get() == null) { + processed.set(newChars); + } else { + processed.set(addAll(processed.get(), newChars)); + } + } + } + + return result.buffer.length(); + } + } + + public boolean canRead(int sz) { + synchronized (processed) { + if (processed.get() == null) return false; + return eof || readPos + sz < processed.get().length; + } + } + + public int read(char[] cbuf, int off, int len) { + synchronized (processed) { + final char[] buf = processed.get(); + + // nothing to read + if (buf == null || buf.length == 0) { + return eof ? -1 : 0; + } + + // do we have enough to fill? + if (readPos + len < buf.length) { + System.arraycopy(buf, readPos, cbuf, off, len); + readPos += len; + return len; + } else { + // are we at EOF? if so, return the last bytes + if (eof) { + // are we really AT eof? + if (readPos == buf.length) return -1; + } + // return what we can, which may be nothing if we have a buffer under-run + final int remainingLen = buf.length - readPos; + System.arraycopy(buf, readPos, cbuf, off, remainingLen); + readPos = buf.length; + return remainingLen; + } + } + } + + public String readLine() { + synchronized (processed) { + final StringBuilder buf = new StringBuilder(String.valueOf(processed.get())); + int newline = buf.indexOf("\n", readPos); + final String line; + if (newline == -1 || newline == buf.length() - 1) { + line = buf.substring(readPos, buf.length()); + readPos = buf.length(); + } else { + line = buf.substring(readPos, newline); + readPos = newline + 1; + } + return line; + } + } + } + +} + diff --git a/src/main/java/org/cobbzilla/util/io/regex/RegexFilterResult.java b/src/main/java/org/cobbzilla/util/io/regex/RegexFilterResult.java new file mode 100644 index 0000000..5baf73b --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/regex/RegexFilterResult.java @@ -0,0 +1,11 @@ +package org.cobbzilla.util.io.regex; + +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public class RegexFilterResult { + + public StringBuilder buffer; + public int remainder; + +} diff --git a/src/main/java/org/cobbzilla/util/io/regex/RegexInsertionFilter.java b/src/main/java/org/cobbzilla/util/io/regex/RegexInsertionFilter.java new file mode 100644 index 0000000..91a360a --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/regex/RegexInsertionFilter.java @@ -0,0 +1,86 @@ +package org.cobbzilla.util.io.regex; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static java.util.regex.Pattern.CASE_INSENSITIVE; +import static java.util.regex.Pattern.MULTILINE; + +@NoArgsConstructor @Accessors(chain=true) +public class RegexInsertionFilter implements RegexStreamFilter { + + @Getter @Setter private String pattern; + @Getter @Setter private int flags = CASE_INSENSITIVE | MULTILINE; + @Getter @Setter private int group = 1; + @Getter @Setter private String before; + @Getter @Setter private String after; + + @Getter(lazy=true) private final Pattern _pattern = Pattern.compile(pattern, flags); + + @Override public void configure(JsonNode config) { + + this.pattern = config.get("pattern").textValue(); + + final JsonNode groupNode = config.get("group"); + this.group = groupNode == null ? 1 : groupNode.intValue(); + + final JsonNode replacementNode = config.get("replacement"); + this.after = replacementNode == null ? "" : replacementNode.textValue(); + + final JsonNode afterReplacementNode = config.get("after"); + this.after = afterReplacementNode == null ? "" : afterReplacementNode.textValue(); + + final JsonNode beforeReplacementNode = config.get("before"); + this.before = beforeReplacementNode == null ? "" : beforeReplacementNode.textValue(); + } + + public RegexFilterResult apply(StringBuilder buffer, boolean eof) { + final StringBuilder result = new StringBuilder(buffer.length()); + int start = 0; + final Matcher matcher = get_pattern().matcher(buffer.toString()); + while (matcher.find(start)) { + + // add everything before the first match + result.append(buffer.subSequence(start, matcher.start())); + + // add the before stuff + if (before != null) result.append(before); + + // add the group match itself + result.append(buffer.subSequence(matcher.start(group), matcher.end(group))); + + // add the after stuff + if (after != null) result.append(after); + + // add everything after the group match + result.append(buffer.subSequence(matcher.end(group), matcher.end())); + + // advance start pointer and track last match end + start = matcher.end(); + } + + if (eof) { + // we are at the end, include everything else, no remainder + result.append(buffer.subSequence(start, buffer.length())); + return new RegexFilterResult(result, 0); + } + + // nothing matched + // leave 1k remaining to reprocess, we might see our pattern again. + final int totalRemainder = buffer.length() - start; + if (totalRemainder > 1024) { + result.append(buffer.subSequence(start, buffer.length()-1024)); + return new RegexFilterResult(result, 1024); + } else { + // leave the entire remainder, we can't be sure + return new RegexFilterResult(result, totalRemainder); + } + } + +} \ No newline at end of file diff --git a/src/main/java/org/cobbzilla/util/io/regex/RegexReplacementFilter.java b/src/main/java/org/cobbzilla/util/io/regex/RegexReplacementFilter.java new file mode 100644 index 0000000..1c4fe8c --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/regex/RegexReplacementFilter.java @@ -0,0 +1,57 @@ +package org.cobbzilla.util.io.regex; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@NoArgsConstructor +public class RegexReplacementFilter implements RegexStreamFilter { + + @Getter @Setter private Pattern pattern; + @Getter @Setter private int group; + @Getter @Setter private String replacement; + + public RegexReplacementFilter(String regex, int group, String replacement) { + this.pattern = Pattern.compile(regex); + this.group = group; + this.replacement = replacement; + } + + @Override public void configure(JsonNode config) { + this.pattern = Pattern.compile(config.get("pattern").textValue()); + + final JsonNode groupNode = config.get("group"); + this.group = groupNode == null ? 1 : groupNode.intValue(); + + final JsonNode replacementNode = config.get("replacement"); + this.replacement = replacementNode == null ? "" : replacementNode.textValue(); + } + + public RegexFilterResult apply(StringBuilder buffer, boolean eof) { + final StringBuilder result = new StringBuilder(buffer.length()); + int start = 0; + final Matcher matcher = pattern.matcher(buffer.toString()); + while (matcher.find(start)) { + // add everything before the first match + result.append(buffer.subSequence(start, matcher.start())); + + // add everything before the group match + result.append(buffer.subSequence(matcher.start(), matcher.start(group))); + + // add the replacement + result.append(replacement); + + // add everything after the group match + result.append(buffer.subSequence(matcher.end(group), matcher.end())); + + // advance start pointer and track last match end + start = matcher.end(); + } + return new RegexFilterResult(result, 0); + } + +} \ No newline at end of file diff --git a/src/main/java/org/cobbzilla/util/io/regex/RegexStreamFilter.java b/src/main/java/org/cobbzilla/util/io/regex/RegexStreamFilter.java new file mode 100644 index 0000000..ceefadd --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/regex/RegexStreamFilter.java @@ -0,0 +1,11 @@ +package org.cobbzilla.util.io.regex; + +import com.fasterxml.jackson.databind.JsonNode; + +public interface RegexStreamFilter { + + default void configure (JsonNode config) {} + + RegexFilterResult apply(StringBuilder buffer, boolean eof); + +} diff --git a/src/main/java/org/cobbzilla/util/io/regex/RegexStreamFilterResult.java b/src/main/java/org/cobbzilla/util/io/regex/RegexStreamFilterResult.java new file mode 100644 index 0000000..c9d19c4 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/io/regex/RegexStreamFilterResult.java @@ -0,0 +1,10 @@ +package org.cobbzilla.util.io.regex; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@AllArgsConstructor +class RegexStreamFilterResult { + @Getter private final StringBuilder result; + @Getter private final int lastMatchEnd; +} diff --git a/src/main/java/org/cobbzilla/util/javascript/JsEngine.java b/src/main/java/org/cobbzilla/util/javascript/JsEngine.java new file mode 100644 index 0000000..4025f48 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/javascript/JsEngine.java @@ -0,0 +1,172 @@ +package org.cobbzilla.util.javascript; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.oracle.truffle.js.scriptengine.GraalJSScriptEngine; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.HostAccess; + +import javax.script.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.json.JsonUtil.fromJsonOrDie; + +@Accessors(chain=true) @Slf4j +public class JsEngine { + + private final List availableScriptEngines; + + private int maxEngines; + private String defaultScript; + public String getDefaultScript () { return empty(defaultScript) ? "" : defaultScript; } + + public JsEngine() { this(new JsEngineConfig(1, 1, null)); } + + public JsEngine(JsEngineConfig config) { + availableScriptEngines = new ArrayList<>(config.getMinEngines()); + maxEngines = config.getMaxEngines(); + defaultScript = config.getDefaultScript(); + for (int i=0; i true)); + if (report) log.info("getEngine: creating scripting engine #"+ engCounter.incrementAndGet()+" ("+availableScriptEngines.size()+" available, "+inUse.get()+" in use)"); + return engine; + } + + public T evaluate(String code, Map context) { + ScriptEngine engine = null; + final int numEngines; + synchronized (availableScriptEngines) { + numEngines = availableScriptEngines.size(); + if (numEngines > 0) { + engine = availableScriptEngines.remove(0); + } + } + if (engine == null) { + if (numEngines >= maxEngines) return die("evaluate("+code+"): maxEngines ("+maxEngines+") reached, no js engines available to execute, "+inUse.get()+" in use"); + engine = getEngine(); + } + inUse.incrementAndGet(); + + try { + final ScriptContext scriptContext = new SimpleScriptContext(); + final SimpleBindings bindings = new SimpleBindings(); + for (Map.Entry entry : context.entrySet()) { + final Object value = entry.getValue(); + final Object wrappedOut; + Object[] wrappedArray = null; + if (value == null) { + wrappedOut = null; + } else if (value instanceof JsWrappable) { + wrappedOut = ((JsWrappable) value).jsObject(); + } else if (value instanceof ArrayNode) { + wrappedOut = fromJsonOrDie((JsonNode) value, Object[].class); + } else if (value instanceof JsonNode) { + wrappedOut = fromJsonOrDie((JsonNode) value, Object.class); + } else if (value.getClass().isArray()) { + wrappedArray = (Object[]) value; + wrappedOut = null; + } else { + wrappedOut = value; + } + if (wrappedArray != null) { + bindings.put(entry.getKey(), wrappedArray); + } else { + bindings.put(entry.getKey(), wrappedOut); + } + } + scriptContext.setBindings(bindings, ScriptContext.ENGINE_SCOPE); + try { + Object eval = engine.eval(getDefaultScript()+"\n"+code, scriptContext); + return (T) eval; + } catch (ScriptException e) { + throw new IllegalStateException(e); + } + } finally { + synchronized (availableScriptEngines) { + availableScriptEngines.add(engine); + } + inUse.decrementAndGet(); + } + } + + public boolean evaluateBoolean(String code, Map ctx) { + final Object result = evaluate(code, ctx); + return result == null ? false : Boolean.valueOf(result.toString().toLowerCase()); + } + + public boolean evaluateBoolean(String code, Map ctx, boolean defaultValue) { + try { + return evaluateBoolean(code, ctx); + } catch (Exception e) { + log.debug("evaluateBoolean: returning "+defaultValue+" due to exception:"+e); + return defaultValue; + } + } + + public Integer evaluateInt(String code, Map ctx) { + final Object result = evaluate(code, ctx); + if (result == null) return null; + if (result instanceof Number) return ((Number) result).intValue(); + return Integer.parseInt(result.toString().trim()); + } + + public Long evaluateLong(String code, Map ctx) { + final Object result = evaluate(code, ctx); + if (result == null) return null; + if (result instanceof Number) return ((Number) result).longValue(); + return Long.parseLong(result.toString().trim()); + } + + public Long evaluateLong(String code, Map ctx, Long defaultValue) { + try { + return evaluateLong(code, ctx); + } catch (Exception e) { + log.debug("evaluateLong: returning "+defaultValue+" due to exception:"+e); + return defaultValue; + } + } + + public String evaluateString(String condition, Map ctx) { + final Object rval = evaluate(condition, ctx); + if (rval == null) return null; + + if (rval instanceof String) return rval.toString(); + if (rval instanceof Number) { + if (rval.toString().endsWith(".0")) return ""+((Number) rval).longValue(); + return rval.toString(); + } + return rval.toString(); + } + + public String functionOfX(String value, String script) { + final Map ctx = new HashMap<>(); + ctx.put("x", value); + try { + return String.valueOf(evaluateInt(script, ctx)); + } catch (Exception e) { + log.warn("functionOfX('"+value+"', '"+script+"', NOT applying due to exception: "+e); + return value; + } + } +} diff --git a/src/main/java/org/cobbzilla/util/javascript/JsEngineConfig.java b/src/main/java/org/cobbzilla/util/javascript/JsEngineConfig.java new file mode 100644 index 0000000..7ba8580 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/javascript/JsEngineConfig.java @@ -0,0 +1,16 @@ +package org.cobbzilla.util.javascript; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; + +@NoArgsConstructor @AllArgsConstructor @Accessors(chain=true) +public class JsEngineConfig { + + @Getter @Setter private int minEngines; + @Getter @Setter private int maxEngines; + @Getter @Setter private String defaultScript; + +} diff --git a/src/main/java/org/cobbzilla/util/javascript/JsEngineFactory.java b/src/main/java/org/cobbzilla/util/javascript/JsEngineFactory.java new file mode 100644 index 0000000..6c70cd7 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/javascript/JsEngineFactory.java @@ -0,0 +1,7 @@ +package org.cobbzilla.util.javascript; + +public interface JsEngineFactory { + + JsEngine getJs (); + +} diff --git a/src/main/java/org/cobbzilla/util/javascript/JsWrappable.java b/src/main/java/org/cobbzilla/util/javascript/JsWrappable.java new file mode 100644 index 0000000..ae0d7a6 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/javascript/JsWrappable.java @@ -0,0 +1,7 @@ +package org.cobbzilla.util.javascript; + +public interface JsWrappable { + + Object jsObject(); + +} diff --git a/src/main/java/org/cobbzilla/util/javascript/StandardJsEngine.java b/src/main/java/org/cobbzilla/util/javascript/StandardJsEngine.java new file mode 100644 index 0000000..7c89aa1 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/javascript/StandardJsEngine.java @@ -0,0 +1,37 @@ +package org.cobbzilla.util.javascript; + +import lombok.extern.slf4j.Slf4j; + +import java.util.HashMap; +import java.util.Map; + +import static org.cobbzilla.util.io.StreamUtil.stream2string; +import static org.cobbzilla.util.string.StringUtil.getPackagePath; + +@Slf4j +public class StandardJsEngine extends JsEngine { + + public StandardJsEngine() { this(1, 1); } + + public StandardJsEngine(int minEngines, int maxEngines) { super(new JsEngineConfig(minEngines, maxEngines, STANDARD_FUNCTIONS)); } + + public static final String STANDARD_FUNCTIONS = stream2string(getPackagePath(StandardJsEngine.class)+"/standard_js_lib.js"); + + private static final String ESC_DOLLAR = "__ESCAPED_DOLLAR_SIGN__"; + public static String replaceDollarSigns(String val) { + return val.replace("'$", ESC_DOLLAR) + .replaceAll("(\\$(\\d+(\\.\\d{2})?))", "($2 * 100)") + .replace(ESC_DOLLAR, "'$"); + } + + public String round(String value, String script) { + final Map ctx = new HashMap<>(); + ctx.put("x", value); + try { + return String.valueOf(evaluateInt(script, ctx)); + } catch (Exception e) { + log.warn("round('"+value+"', '"+script+"', NOT rounding due to exception: "+e); + return value; + } + } +} diff --git a/src/main/java/org/cobbzilla/util/jdbc/DbDumpMode.java b/src/main/java/org/cobbzilla/util/jdbc/DbDumpMode.java new file mode 100644 index 0000000..9e8ca18 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/jdbc/DbDumpMode.java @@ -0,0 +1,11 @@ +package org.cobbzilla.util.jdbc; + +import com.fasterxml.jackson.annotation.JsonCreator; + +public enum DbDumpMode { + + all, schema, data, pre_data, post_data; + + @JsonCreator public static DbDumpMode fromString (String val) { return valueOf(val.toLowerCase()); } + +} diff --git a/src/main/java/org/cobbzilla/util/jdbc/DbUrlUtil.java b/src/main/java/org/cobbzilla/util/jdbc/DbUrlUtil.java new file mode 100644 index 0000000..3b596af --- /dev/null +++ b/src/main/java/org/cobbzilla/util/jdbc/DbUrlUtil.java @@ -0,0 +1,17 @@ +package org.cobbzilla.util.jdbc; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class DbUrlUtil { + + public static final Pattern JDBC_URL_REGEX = Pattern.compile("^jdbc:postgresql://[\\.\\w]+:\\d+/(.+)$"); + + public static String setDbName(String url, String dbName) { + final Matcher matcher = JDBC_URL_REGEX.matcher(url); + if (!matcher.find()) return url; + final String renamed = matcher.replaceFirst(dbName); + return renamed; + } + +} diff --git a/src/main/java/org/cobbzilla/util/jdbc/DebugConnection.java b/src/main/java/org/cobbzilla/util/jdbc/DebugConnection.java new file mode 100644 index 0000000..3b28da2 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/jdbc/DebugConnection.java @@ -0,0 +1,26 @@ +package org.cobbzilla.util.jdbc; + +import lombok.Delegate; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; + +import java.sql.Connection; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class DebugConnection implements Connection { + + private static final AtomicInteger counter = new AtomicInteger(0); + + private int id; + + @Delegate private Connection delegate; + + public DebugConnection(Connection delegate) { + this.id = counter.getAndIncrement(); + this.delegate = delegate; + final String msg = "DebugConnection " + id + " opened from " + ExceptionUtils.getStackTrace(new Exception("opened")); + log.info(msg); + } + +} diff --git a/src/main/java/org/cobbzilla/util/jdbc/DebugDriver.java b/src/main/java/org/cobbzilla/util/jdbc/DebugDriver.java new file mode 100644 index 0000000..cedf5c7 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/jdbc/DebugDriver.java @@ -0,0 +1,11 @@ +package org.cobbzilla.util.jdbc; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Properties; + +public interface DebugDriver { + + public Connection connect(String url, Properties info) throws SQLException; + +} diff --git a/src/main/java/org/cobbzilla/util/jdbc/DebugPostgresqlDriver.java b/src/main/java/org/cobbzilla/util/jdbc/DebugPostgresqlDriver.java new file mode 100644 index 0000000..82f46f0 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/jdbc/DebugPostgresqlDriver.java @@ -0,0 +1,40 @@ +package org.cobbzilla.util.jdbc; + +import lombok.Delegate; +import lombok.extern.slf4j.Slf4j; + +import java.sql.Connection; +import java.sql.Driver; +import java.sql.SQLException; +import java.util.Properties; + +import static org.cobbzilla.util.reflect.ReflectionUtil.instantiate; + +@Slf4j +public class DebugPostgresqlDriver implements Driver, DebugDriver { + + private static final String DEBUG_PREFIX = "debug:"; + private static final String POSTGRESQL_PREFIX = "jdbc:postgresql:"; + private static final String DRIVER_CLASS_NAME = "org.postgresql.Driver"; + + static { + try { + java.sql.DriverManager.registerDriver(new DebugPostgresqlDriver()); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Delegate(excludes = DebugDriver.class) + private Driver driver = instantiate(DRIVER_CLASS_NAME); + + @Override public Connection connect(String url, Properties info) throws SQLException { + if (url.startsWith(DEBUG_PREFIX)) { + url = url.substring(DEBUG_PREFIX.length()); + if (url.startsWith(POSTGRESQL_PREFIX)) { + return new DebugConnection(driver.connect(url, info)); + } + } + throw new IllegalArgumentException("can't connect: "+url); + } +} diff --git a/src/main/java/org/cobbzilla/util/jdbc/ResultSetBean.java b/src/main/java/org/cobbzilla/util/jdbc/ResultSetBean.java new file mode 100644 index 0000000..67eeed2 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/jdbc/ResultSetBean.java @@ -0,0 +1,101 @@ +package org.cobbzilla.util.jdbc; + +import lombok.AccessLevel; +import lombok.Cleanup; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.sql.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +@NoArgsConstructor(access=AccessLevel.PRIVATE) @Slf4j +public class ResultSetBean { + + public static final ResultSetBean EMPTY = new ResultSetBean(); + + @Getter private final ArrayList> rows = new ArrayList<>(); + public boolean isEmpty () { return rows.isEmpty(); } + + public int rowCount () { return isEmpty() ? 0 : rows.size(); } + + public Map first () { return rows.get(0); } + public Integer count () { return isEmpty() ? null : Integer.valueOf(rows.get(0).entrySet().iterator().next().getValue().toString()); } + public int countOrZero () { return isEmpty() ? 0 : Integer.parseInt(rows.get(0).entrySet().iterator().next().getValue().toString()); } + + public ResultSetBean (ResultSet rs) throws SQLException { rows.addAll(read(rs)); } + public ResultSetBean (PreparedStatement ps) throws SQLException { rows.addAll(read(ps)); } + public ResultSetBean (Connection conn, String sql) throws SQLException { rows.addAll(read(conn, sql)); } + + private final AtomicReference rsMetaData = new AtomicReference<>(); + public ResultSetMetaData getRsMetaData(ResultSet rs) throws SQLException { + if (rsMetaData.get() == null) { + synchronized (rsMetaData) { + if (rsMetaData.get() == null) { + rsMetaData.set(rs.getMetaData()); + } + } + } + return rsMetaData.get(); + } + public ResultSetMetaData getRsMetaData() { return rsMetaData.get(); } + + private List> read(Connection conn, String sql) throws SQLException { + @Cleanup final PreparedStatement ps = conn.prepareStatement(sql); + return read(ps); + } + private List> read(PreparedStatement ps) throws SQLException { + @Cleanup final ResultSet rs = ps.executeQuery(); + return read(rs); + } + + private List> read(ResultSet rs) throws SQLException { + final ResultSetMetaData rsMetaData = getRsMetaData(rs); + final int numColumns = rsMetaData.getColumnCount(); + final List> results = new ArrayList<>(); + while (rs.next()){ + final HashMap row = row2map(rs, rsMetaData, numColumns); + results.add(row); + } + return results; + } + + public List getColumnValues (String column) { + final List values = new ArrayList<>(); + for (Map row : getRows()) values.add((T) row.get(column)); + return values; + } + + public static HashMap row2map(ResultSet rs) throws SQLException { + final ResultSetMetaData rsMetaData = rs.getMetaData(); + final int numColumns = rsMetaData.getColumnCount(); + return row2map(rs, rsMetaData, numColumns); + } + + public static HashMap row2map(ResultSet rs, ResultSetMetaData rsMetaData) throws SQLException { + final int numColumns = rsMetaData.getColumnCount(); + return row2map(rs, rsMetaData, numColumns); + } + + public static HashMap row2map(ResultSet rs, ResultSetMetaData rsMetaData, int numColumns) throws SQLException { + final HashMap row = new HashMap<>(numColumns); + for(int i=1; i<=numColumns; ++i){ + row.put(rsMetaData.getColumnName(i), rs.getObject(i)); + } + return row; + } + + public static List getColumns(ResultSetMetaData rsMetaData) throws SQLException { + int columnCount = rsMetaData.getColumnCount(); + final List columns = new ArrayList<>(columnCount); + for (int i=1; i<=columnCount; ++i) { + columns.add(rsMetaData.getColumnName(i)); + } + return columns; + } + +} diff --git a/src/main/java/org/cobbzilla/util/jdbc/TypedResultSetBean.java b/src/main/java/org/cobbzilla/util/jdbc/TypedResultSetBean.java new file mode 100644 index 0000000..7186524 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/jdbc/TypedResultSetBean.java @@ -0,0 +1,83 @@ +package org.cobbzilla.util.jdbc; + +import lombok.AccessLevel; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.reflect.ReflectionUtil; + +import java.lang.reflect.Field; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; + +import static org.cobbzilla.util.reflect.ReflectionUtil.getDeclaredField; +import static org.cobbzilla.util.reflect.ReflectionUtil.instantiate; +import static org.cobbzilla.util.string.StringUtil.snakeCaseToCamelCase; + +@Slf4j +public class TypedResultSetBean extends ResultSetBean implements Iterable { + + public TypedResultSetBean(Class clazz, ResultSet rs) throws SQLException { super(rs); rowType = clazz; } + public TypedResultSetBean(Class clazz, PreparedStatement ps) throws SQLException { super(ps); rowType = clazz; } + public TypedResultSetBean(Class clazz, Connection conn, String sql) throws SQLException { super(conn, sql); rowType = clazz; } + + private final Class rowType; + @Getter(lazy=true, value=AccessLevel.PRIVATE) private final List typedRows = getTypedRows(rowType); + + @Override public Iterator iterator() { return new ArrayList<>(getTypedRows()).iterator(); } + + private List getTypedRows(Class clazz) { + final List typedRows = new ArrayList<>(); + for (Map row : getRows()) { + final T thing = instantiate(clazz); + for (String name : row.keySet()) { + final String field = snakeCaseToCamelCase(name); + try { + final Object value = row.get(name); + readField(thing, field, value); + } catch (Exception e) { + log.warn("getTypedRows: error setting "+field+": "+e); + } + } + typedRows.add(thing); + } + return typedRows; + } + + protected void readField(T thing, String field, Object value) { + if (value != null) { + try { + ReflectionUtil.set(thing, field, value); + } catch (Exception e) { + // try field setter + try { + final Field f = getDeclaredField(thing.getClass(), field); + if (f != null) { + f.setAccessible(true); + f.set(thing, value); + } else { + log.warn("readField: field "+thing.getClass().getName()+"."+field+" not found via setter nor via field: "+e); + } + } catch (Exception e2) { + log.warn("readField: field "+thing.getClass().getName()+"."+field+" not found via setter nor via field: "+e2); + } + } + } + } + + public Map map (String field) { + final Map map = new HashMap<>(); + for (T thing : this) { + map.put((K) ReflectionUtil.get(thing, field), thing); + } + return map; + } + + public T firstObject() { + final Iterator iter = iterator(); + return iter.hasNext() ? iter.next() : null; + } + +} diff --git a/src/main/java/org/cobbzilla/util/jdbc/UncheckedSqlException.java b/src/main/java/org/cobbzilla/util/jdbc/UncheckedSqlException.java new file mode 100644 index 0000000..e56b3b6 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/jdbc/UncheckedSqlException.java @@ -0,0 +1,23 @@ +package org.cobbzilla.util.jdbc; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.sql.SQLException; + +@AllArgsConstructor +public class UncheckedSqlException extends RuntimeException { + + @Getter private final SQLException sqlException; + + @Override public String getMessage() { return sqlException.getMessage(); } + + @Override public String getLocalizedMessage() { return sqlException.getLocalizedMessage(); } + + @Override public synchronized Throwable getCause() { return sqlException.getCause(); } + + @Override public synchronized Throwable initCause(Throwable throwable) { return sqlException.initCause(throwable); } + + @Override public String toString() { return sqlException.toString(); } + +} diff --git a/src/main/java/org/cobbzilla/util/json/JsonEdit.java b/src/main/java/org/cobbzilla/util/json/JsonEdit.java new file mode 100644 index 0000000..2c142b4 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/json/JsonEdit.java @@ -0,0 +1,229 @@ +package org.cobbzilla.util.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.ValueNode; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.net.URL; +import java.util.*; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.json.JsonUtil.*; +import static org.cobbzilla.util.json.JsonUtil.FULL_MAPPER_ALLOW_COMMENTS; + +/** + * Facilitates editing JSON files. + * + * Notes: + * - only one read operation can be specified. the 'edit' method will return the value of the first read operation processed. + * - if you write a node that does not exist, it will be created (if it can be) + */ +@Accessors(chain=true) +public class JsonEdit { + + public static final ObjectMapper JSON = FULL_MAPPER_ALLOW_COMMENTS; + + @Getter @Setter private Object jsonData; + @Getter @Setter private List operations = new ArrayList<>(); + + public JsonEdit addOperation (JsonEditOperation operation) { operations.add(operation); return this; } + + public String edit () throws Exception { + JsonNode root = readJson(); + for (JsonEditOperation operation : operations) { + if (operation.isRead()) return JsonUtil.toString(findNode(root, operation.getPath())); + root = apply(root, operation); + } + return JsonUtil.toString(JSON.treeToValue(root, Object.class)); + } + + private JsonNode readJson() throws IOException { + if (jsonData instanceof JsonNode) return (JsonNode) jsonData; + if (jsonData instanceof InputStream) return JSON.readTree((InputStream) jsonData); + if (jsonData instanceof Reader) return JSON.readTree((Reader) jsonData); + if (jsonData instanceof String) return JSON.readTree((String) jsonData); + if (jsonData instanceof File) return JSON.readTree((File) jsonData); + if (jsonData instanceof URL) return JSON.readTree((URL) jsonData); + throw new IllegalArgumentException("jsonData is not a JsonNode, InputStream, Reader, String, File or URL"); + } + + private JsonNode apply(JsonNode root, JsonEditOperation operation) throws IOException { + final List path = findNodePath(root, operation.getPath()); + + switch (operation.getType()) { + case write: + root = write(root, path, operation); + break; + + case delete: + delete(path, operation); + break; + + case sort: + root = sort(root, path, operation); + break; + + default: throw new IllegalArgumentException("unsupported operation: "+operation.getType()); + } + return root; + } + + private JsonNode write(JsonNode root, List path, JsonEditOperation operation) throws IOException { + + JsonNode current = path.get(path.size()-1); + final JsonNode parent = path.size() > 1 ? path.get(path.size()-2) : null; + final JsonNode data = operation.getNode(); + + if (current == MISSING) { + // add a new node to parent + return addToParent(root, operation, path); + } + + if (current instanceof ObjectNode) { + final ObjectNode node = (ObjectNode) current; + if (data instanceof ObjectNode) { + final Iterator> fields = data.fields(); + while (fields.hasNext()) { + final Map.Entry entry = fields.next(); + node.set(entry.getKey(), entry.getValue()); + } + } else { + return addToParent(root, operation, path); + } + + } else if (current instanceof ArrayNode) { + final ArrayNode node = (ArrayNode) current; + if (operation.hasIndex()) { + node.set(operation.getIndex(), operation.getNode()); + } else { + node.add(operation.getNode()); + } + + } else if (current instanceof ValueNode) { + if (parent == null) return newObjectNode().set(operation.getName(), data); + + // overwrite value node at location + addToParent(root, operation, path); + + } else { + throw new IllegalArgumentException("Cannot append to node (is a "+current.getClass().getName()+"): "+current); + } + + return root; + } + + private JsonNode addToParent(JsonNode root, JsonEditOperation operation, List path) throws IOException { + + JsonNode parent = path.size() > 1 ? path.get(path.size()-2) : null; + final JsonNode data = operation.getNode(); + + if (parent == null) return newObjectNode().set(operation.getName(), data); + + if (parent instanceof ObjectNode) { + // more than one missing node? + while (path.size() <= operation.getNumPathSegments()) { + final String childName = operation.getName(path.size()-2); + final ObjectNode newNode = newObjectNode(); + ((ObjectNode) parent).set(childName, newNode); + + // re-generate path now that we've created one missing parent + path = findNodePath(root, operation.getPath()); + parent = newNode; + } + + // creating a new array node under parent? + if (operation.isEmptyBrackets()) { + final ArrayNode newArrayNode = newArrayNode(); + ((ObjectNode) parent).set(operation.getName(), newArrayNode); + newArrayNode.add(data); + + } else { + // otherwise, just set a field on the parent object + ((ObjectNode) parent).set(operation.getName(), data); + } + + } else if (parent instanceof ArrayNode) { + if (operation.isEmptyBrackets()) { + ((ArrayNode) parent).add(data); + } else { + ((ArrayNode) parent).set(operation.getIndex(), data); + } + } else { + throw new IllegalArgumentException("Cannot append to node (is a "+parent.getClass().getName()+"): "+parent); + } + + return root; + } + + private void delete(List path, JsonEditOperation operation) { + if (path.size() < 2) throw new IllegalArgumentException("Cannot delete root"); + final JsonNode parent = path.get(path.size()-2); + + if (parent instanceof ArrayNode) { + ((ArrayNode) parent).remove(operation.getIndex()); + + } else if (parent instanceof ObjectNode) { + ((ObjectNode) parent).remove(operation.getName()); + + } else { + throw new IllegalArgumentException("Cannot remove node (parent is a "+parent.getClass().getName()+")"); + } + } + + private JsonNode sort(JsonNode root, List path, JsonEditOperation operation) { + final JsonNode array = path.get(path.size()-1); + final JsonNode parent = path.size() == 1 ? array : path.get(path.size()-2); + if (!array.isArray()) return die("sort: "+operation.getPath()+" is not an array: "+json(array)); + + // sort array + final SortedSet sorted = new TreeSet<>(new JsonNodeComparator(operation.getJson())); + final Iterator iter = array.iterator(); + while (iter.hasNext()) sorted.add(iter.next()); + + // create a new ArrayNode for the sorted array + final ArrayNode newArray = newArrayNode(); + for (JsonNode n : sorted) newArray.add(n); + + // if we are sorting a root-level array, just return it now + if (parent == array) return newArray; + + // find previous array node, remove it and add new one + boolean replaced = false; + if (parent.isArray()) { + final ArrayNode parentArray = (ArrayNode) parent; + for (int i=0; i fieldNames = parentObject.fieldNames(); + while (fieldNames.hasNext()) { + final String fieldName = fieldNames.next(); + final JsonNode n = parentObject.get(fieldName); + if (n == array) { + parentObject.remove(fieldName); + parentObject.set(fieldName, newArray); + replaced = true; + break; + } + } + } + if (!replaced) return die("sort: error replacing original array with sorted array"); + return root; + } + +} diff --git a/src/main/java/org/cobbzilla/util/json/JsonEditOperation.java b/src/main/java/org/cobbzilla/util/json/JsonEditOperation.java new file mode 100644 index 0000000..747e0df --- /dev/null +++ b/src/main/java/org/cobbzilla/util/json/JsonEditOperation.java @@ -0,0 +1,68 @@ +package org.cobbzilla.util.json; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; + +import java.io.IOException; +import java.util.List; + +import static org.cobbzilla.util.json.JsonUtil.FULL_MAPPER; + +@Accessors(chain=true) +public class JsonEditOperation { + + @Getter @Setter private JsonEditOperationType type; + @Getter @Setter private String path; + @Getter @Setter private String json; + + public boolean isRead() { return type == JsonEditOperationType.read; } + + public JsonNode getNode () throws IOException { return FULL_MAPPER.readTree(json); } + + public boolean hasIndex () { return getIndex() != null; } + + @JsonIgnore @Getter(lazy=true) private final List tokens = initTokens(); + private List initTokens() { return JsonUtil.tokenize(path); } + + public boolean isEmptyBrackets () { + int bracketPos = path.indexOf("["); + int bracketClosePos = path.indexOf("]"); + return bracketPos != -1 && bracketClosePos != -1 && bracketClosePos == bracketPos+1; + } + + public Integer getIndex() { + List tokens = getTokens(); + if (tokens.size() <= 1) return index(path); + return index(tokens.get(tokens.size()-1)); + } + + private Integer index(String path) { + try { + int bracketPos = path.indexOf("["); + int bracketClosePos = path.indexOf("]"); + if (bracketPos != -1 && bracketClosePos != -1 && bracketClosePos > bracketPos) { + return Integer.valueOf(path.substring(bracketPos + 1, bracketClosePos)); + } + } catch (Exception ignored) {} + return null; + } + + public String getName() { + final List tokens = getTokens(); + if (tokens.size() <= 1) return stripEmptyTrailingBrackets(path); + return stripEmptyTrailingBrackets(tokens.get(tokens.size() - 1)); + } + + private String stripEmptyTrailingBrackets(String path) { + return path.endsWith("[]") ? path.substring(0, path.length()-2) : path; + } + + public int getNumPathSegments() { return getTokens().size(); } + + public String getName(int part) { + return getTokens().get(part); } + +} diff --git a/src/main/java/org/cobbzilla/util/json/JsonEditOperationType.java b/src/main/java/org/cobbzilla/util/json/JsonEditOperationType.java new file mode 100644 index 0000000..cf08937 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/json/JsonEditOperationType.java @@ -0,0 +1,11 @@ +package org.cobbzilla.util.json; + +import com.fasterxml.jackson.annotation.JsonCreator; + +public enum JsonEditOperationType { + + read, write, delete, sort; + + @JsonCreator public static JsonEditOperationType create(String value) { return valueOf(value.toLowerCase()); } + +} diff --git a/src/main/java/org/cobbzilla/util/json/JsonNodeComparator.java b/src/main/java/org/cobbzilla/util/json/JsonNodeComparator.java new file mode 100644 index 0000000..b7ddb68 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/json/JsonNodeComparator.java @@ -0,0 +1,22 @@ +package org.cobbzilla.util.json; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.AllArgsConstructor; + +import java.util.Comparator; + +import static org.cobbzilla.util.json.JsonUtil.fromJsonOrDie; + +@AllArgsConstructor +public class JsonNodeComparator implements Comparator { + + final String path; + + @Override public int compare(JsonNode n1, JsonNode n2) { + final JsonNode v1 = fromJsonOrDie(n1, path, JsonNode.class); + final JsonNode v2 = fromJsonOrDie(n2, path, JsonNode.class); + if (v1 == null) return 1; + if (v2 == null) return -1; + return v1.textValue().compareTo(v2.textValue()); + } +} diff --git a/src/main/java/org/cobbzilla/util/json/JsonPathNotFoundException.java b/src/main/java/org/cobbzilla/util/json/JsonPathNotFoundException.java new file mode 100644 index 0000000..676b501 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/json/JsonPathNotFoundException.java @@ -0,0 +1,9 @@ +package org.cobbzilla.util.json; + +public class JsonPathNotFoundException extends RuntimeException { + + public JsonPathNotFoundException(String message) { + super(message); + } + +} diff --git a/src/main/java/org/cobbzilla/util/json/JsonSerializableException.java b/src/main/java/org/cobbzilla/util/json/JsonSerializableException.java new file mode 100644 index 0000000..5222d76 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/json/JsonSerializableException.java @@ -0,0 +1,26 @@ +package org.cobbzilla.util.json; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; +import org.apache.commons.lang3.exception.ExceptionUtils; + +@NoArgsConstructor @Accessors(chain=true) +public class JsonSerializableException { + + @Getter @Setter private String exceptionClass; + @Getter @Setter private String message; + @Getter @Setter private String stackTrace; + + public JsonSerializableException (Throwable t) { + exceptionClass = t.getClass().getName(); + message = t.getMessage(); + stackTrace = ExceptionUtils.getStackTrace(t); + } + + public String shortString () { return exceptionClass + ": " + getMessage(); } + + public String toString () { return shortString() + "\n" + getStackTrace(); } + +} diff --git a/src/main/java/org/cobbzilla/util/json/JsonUtil.java b/src/main/java/org/cobbzilla/util/json/JsonUtil.java new file mode 100644 index 0000000..c521223 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/json/JsonUtil.java @@ -0,0 +1,527 @@ +package org.cobbzilla.util.json; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.io.JsonStringEncoder; +import com.fasterxml.jackson.core.util.BufferRecyclers; +import com.fasterxml.jackson.databind.*; +import com.fasterxml.jackson.databind.node.*; +import org.cobbzilla.util.io.FileSuffixFilter; +import org.cobbzilla.util.io.FileUtil; +import org.cobbzilla.util.io.FilenameSuffixFilter; +import org.cobbzilla.util.io.StreamUtil; + +import java.io.*; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +import static org.cobbzilla.util.daemon.ZillaRuntime.big; +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +public class JsonUtil { + + public static final String EMPTY_JSON = "{}"; + public static final String EMPTY_JSON_ARRAY = "[]"; + + public static final JsonNode MISSING = MissingNode.getInstance(); + + public static final FileFilter JSON_FILES = new FileSuffixFilter(".json"); + public static final FilenameFilter JSON_FILENAMES = new FilenameSuffixFilter(".json"); + + public static final ObjectMapper COMPACT_MAPPER = new ObjectMapper(); + + public static final ObjectMapper FULL_MAPPER = new ObjectMapper() + .configure(SerializationFeature.INDENT_OUTPUT, true); + + public static final ObjectWriter FULL_WRITER = FULL_MAPPER.writer(); + + public static final ObjectMapper FULL_MAPPER_ALLOW_COMMENTS = new ObjectMapper() + .configure(SerializationFeature.INDENT_OUTPUT, true); + + static { + FULL_MAPPER_ALLOW_COMMENTS.getFactory().enable(JsonParser.Feature.ALLOW_COMMENTS); + } + + public static final ObjectMapper FULL_MAPPER_ALLOW_COMMENTS_AND_UNKNOWN_FIELDS = new ObjectMapper() + .configure(SerializationFeature.INDENT_OUTPUT, true) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + static { + FULL_MAPPER_ALLOW_COMMENTS_AND_UNKNOWN_FIELDS.getFactory().enable(JsonParser.Feature.ALLOW_COMMENTS); + } + + public static final ObjectMapper FULL_MAPPER_ALLOW_UNKNOWN_FIELDS = new ObjectMapper() + .configure(SerializationFeature.INDENT_OUTPUT, true) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + static { + FULL_MAPPER_ALLOW_UNKNOWN_FIELDS.getFactory().enable(JsonParser.Feature.ALLOW_COMMENTS); + } + + public static final ObjectMapper NOTNULL_MAPPER = FULL_MAPPER + .configure(SerializationFeature.WRITE_NULL_MAP_VALUES, false) + .setSerializationInclusion(JsonInclude.Include.NON_NULL); + + public static final ObjectMapper NOTNULL_MAPPER_ALLOW_EMPTY = FULL_MAPPER + .configure(SerializationFeature.WRITE_NULL_MAP_VALUES, false) + .configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false) + .setSerializationInclusion(JsonInclude.Include.NON_NULL); + + public static final ObjectMapper PUBLIC_MAPPER = buildMapper(); + + public static final ObjectWriter PUBLIC_WRITER = buildWriter(PUBLIC_MAPPER, PublicView.class); + + public static ObjectMapper buildMapper() { + return new ObjectMapper() + .configure(MapperFeature.DEFAULT_VIEW_INCLUSION, false) + .configure(SerializationFeature.INDENT_OUTPUT, true) + .configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false) + .setSerializationInclusion(JsonInclude.Include.NON_NULL); + } + + public static ObjectWriter buildWriter(Class view) { + return buildMapper().writerWithView(view); + } + public static ObjectWriter buildWriter(ObjectMapper mapper, Class view) { + return mapper.writerWithView(view); + } + + public static ArrayNode newArrayNode() { return new ArrayNode(FULL_MAPPER.getNodeFactory()); } + public static ObjectNode newObjectNode() { return new ObjectNode(FULL_MAPPER.getNodeFactory()); } + + public static String find(JsonNode array, String name, String value, String returnValue) { + if (array instanceof ArrayNode) { + for (int i=0; i", ">").replace(" ", " ").replace("\n", "
"); + } + + public static class PublicView {} + + public static String toJson (Object o) throws Exception { return toJson(o, NOTNULL_MAPPER); } + + public static String toJson (Object o, ObjectMapper m) throws Exception { return m.writeValueAsString(o); } + + public static String json (Object o) { return toJsonOrDie(o); } + public static String json (Object o, ObjectMapper m) { return toJsonOrDie(o, m); } + + public static String toJsonOrDie (Object o) { + try { return toJson(o); } catch (Exception e) { + return die("toJson: exception writing object ("+o+"): "+e, e); + } + } + + public static String toJsonOrDie (Object o, ObjectMapper m) { + try { return toJson(o, m); } catch (Exception e) { + return die("toJson: exception writing object ("+o+"): "+e, e); + } + } + + public static String toJsonOrErr(Object o) { + try { return toJson(o); } catch (Exception e) { + return e.toString(); + } + } + + private static Map viewWriters = new ConcurrentHashMap<>(); + + protected static ObjectWriter viewWriter(Class jsonView) { + ObjectWriter w = viewWriters.get(jsonView.getName()); + if (w == null) { + w = JsonUtil.NOTNULL_MAPPER.disable(MapperFeature.DEFAULT_VIEW_INCLUSION).writerWithView(jsonView); + viewWriters.put(jsonView.getName(), w); + } + return w; + } + + public static String toJson (Object o, Class jsonView) throws Exception { + return viewWriter(jsonView).writeValueAsString(o); + } + + public static String toJsonOrDie (Object o, Class jsonView) { + try { return toJson(o, jsonView); } catch (Exception e) { + return die("toJson: exception writing object ("+o+"): "+e, e); + } + } + + public static String toJsonOrErr(Object o, Class jsonView) { + try { return toJson(o, jsonView); } catch (Exception e) { + return e.toString(); + } + } + + public static T fromJson(InputStream json, Class clazz) throws Exception { + return fromJson(StreamUtil.toString(json), clazz); + } + + public static T fromJson(InputStream json, Class clazz, ObjectMapper mapper) throws Exception { + return fromJson(StreamUtil.toString(json), clazz, mapper); + } + + public static T fromJson(File json, Class clazz) throws Exception { + return fromJson(FileUtil.toString(json), clazz); + } + + public static T fromJson(File json, Class clazz, ObjectMapper mapper) throws Exception { + return fromJson(FileUtil.toString(json), clazz, mapper); + } + + public static T fromJson(String json, Class clazz) throws Exception { + return fromJson(json, clazz, JsonUtil.FULL_MAPPER); + } + + public static T fromJson(String json, JavaType type) throws Exception { + if (empty(json)) return null; + return JsonUtil.FULL_MAPPER.readValue(json, type); + } + + public static T fromJson(String json, Class clazz, ObjectMapper mapper) throws Exception { + if (empty(json)) return null; + if (clazz == String.class && !(json.startsWith("\"") && json.endsWith("\""))) { + json = "\"" + json + "\""; + } + return mapper.readValue(json, clazz); + } + + public static T fromJsonOrDie(File json, Class clazz) { + return fromJsonOrDie(FileUtil.toStringOrDie(json), clazz); + } + + public static T json(String json, Class clazz) { return fromJsonOrDie(json, clazz); } + + public static T json(String json, Class clazz, ObjectMapper mapper) { return fromJsonOrDie(json, clazz, mapper); } + + public static T json(JsonNode json, Class clazz) { return fromJsonOrDie(json, clazz); } + + public static List json(JsonNode[] json, Class clazz) { + final List list = new ArrayList<>(); + for (JsonNode node : json) list.add(json(node, clazz)); + return list; + } + + public static T jsonWithComments(String json, Class clazz) { return fromJsonOrDie(json, clazz, FULL_MAPPER_ALLOW_COMMENTS); } + public static T jsonWithComments(JsonNode json, Class clazz) { return fromJsonOrDie(json(json), clazz, FULL_MAPPER_ALLOW_COMMENTS); } + + public static T fromJsonOrDie(String json, Class clazz) { + return fromJsonOrDie(json, clazz, FULL_MAPPER); + } + public static T fromJsonOrDie(String json, Class clazz, ObjectMapper mapper) { + if (empty(json)) return null; + try { + return mapper.readValue(json, clazz); + } catch (IOException e) { + return die("fromJsonOrDie: exception while reading: "+json+": "+e, e); + } + } + + public static T fromJson(String json, String path, Class clazz) throws Exception { + return fromJson(FULL_MAPPER.readTree(json), path, clazz); + } + + public static T fromJson(File json, String path, Class clazz) throws Exception { + return fromJson(FULL_MAPPER.readTree(json), path, clazz); + } + + public static T fromJson(JsonNode child, Class childClass) throws Exception { + return fromJson(child, "", childClass); + } + + public static T fromJsonOrDie(JsonNode child, Class childClass) { + return fromJsonOrDie(child, "", childClass); + } + + public static T fromJsonOrDie(JsonNode node, String path, Class clazz) { + return fromJsonOrDie(node, path, clazz, FULL_MAPPER); + } + + public static T fromJson(JsonNode node, String path, Class clazz) throws Exception { + return fromJson(node, path, clazz, FULL_MAPPER); + } + + public static T fromJsonOrDie(JsonNode node, String path, Class clazz, ObjectMapper mapper) { + try { + return fromJson(node, path, clazz, mapper); + } catch (Exception e) { + return die("fromJsonOrDie: exception while reading: "+node+": "+e, e); + } + } + public static T fromJson(JsonNode node, String path, Class clazz, ObjectMapper mapper) throws Exception { + node = findNode(node, path); + return mapper.convertValue(node, clazz); + } + + public static JsonNode findNode(JsonNode node, String path) throws IOException { + if (node == null) return null; + final List nodePath = findNodePath(node, path); + if (nodePath == null || nodePath.isEmpty()) return null; + final JsonNode lastNode = nodePath.get(nodePath.size()-1); + return lastNode == MISSING ? null : lastNode; + } + + public static String toString(Object node) throws JsonProcessingException { + return node == null ? null : FULL_MAPPER.writeValueAsString(node); + } + + public static String nodeValue (JsonNode node, String path) throws IOException { + return fromJsonOrDie(toString(findNode(node, path)), String.class); + } + + public static List findNodePath(JsonNode node, String path) throws IOException { + + final List nodePath = new ArrayList<>(); + nodePath.add(node); + if (empty(path)) return nodePath; + final List pathParts = tokenize(path); + + for (String pathPart : pathParts) { + int index = -1; + int bracketPos = pathPart.indexOf("["); + int bracketClosePos = pathPart.indexOf("]"); + boolean isEmptyBrackets = false; + if (bracketPos != -1 && bracketClosePos != -1 && bracketClosePos > bracketPos) { + if (bracketClosePos == bracketPos+1) { + // ends with [], they mean to append + isEmptyBrackets = true; + } else { + index = Integer.parseInt(pathPart.substring(bracketPos + 1, bracketClosePos)); + } + pathPart = pathPart.substring(0, bracketPos); + } + if (!empty(pathPart)) { + node = node.get(pathPart); + if (node == null) { + nodePath.add(MISSING); + return nodePath; + } + nodePath.add(node); + + } else if (nodePath.size() > 1) { + return die("findNodePath: invalid path: "+path); + } + if (index != -1) { + node = node.get(index); + nodePath.add(node); + + } else if (isEmptyBrackets) { + nodePath.add(MISSING); + return nodePath; + } + } + return nodePath; + } + + public static List tokenize(String path) { + final List pathParts = new ArrayList<>(); + final StringTokenizer st = new StringTokenizer(path, ".'", true); + boolean collectingQuotedToken = false; + StringBuffer pathToken = new StringBuffer(); + while (st.hasMoreTokens()) { + final String token = st.nextToken(); + if (token.equals("'")) { + collectingQuotedToken = !collectingQuotedToken; + + } else if (collectingQuotedToken) { + pathToken.append(token); + + } else if (token.equals(".") && pathToken.length() > 0) { + pathParts.add(pathToken.toString()); + pathToken = new StringBuffer(); + + } else { + pathToken.append(token); + } + } + if (collectingQuotedToken) throw new IllegalArgumentException("Unterminated single quote in: "+path); + if (pathToken.length() > 0) pathParts.add(pathToken.toString()); + return pathParts; + } + + public static ObjectNode replaceNode(File file, String path, String replacement) throws Exception { + return replaceNode((ObjectNode) FULL_MAPPER.readTree(file), path, replacement); + } + + public static ObjectNode replaceNode(String json, String path, String replacement) throws Exception { + return replaceNode((ObjectNode) FULL_MAPPER.readTree(json), path, replacement); + } + + public static ObjectNode replaceNode(ObjectNode document, String path, String replacement) throws Exception { + + final String simplePath = path.contains(".") ? path.substring(path.lastIndexOf(".")+1) : path; + Integer index = null; + if (simplePath.contains("[")) { + index = Integer.parseInt(simplePath.substring(simplePath.indexOf("[")+1, simplePath.indexOf("]"))); + } + final List found = findNodePath(document, path); + if (found == null || found.isEmpty() || found.get(found.size()-1).equals(MISSING)) { + throw new IllegalArgumentException("path not found: "+path); + } + + final JsonNode parent = found.size() > 1 ? found.get(found.size()-2) : document; + if (index != null) { + final JsonNode origNode = ((ArrayNode) parent).get(index); + ((ArrayNode) parent).set(index, getValueNode(origNode, path, replacement)); + } else { + // what is the original node type? + final JsonNode origNode = parent.get(simplePath); + ((ObjectNode) parent).set(simplePath, getValueNode(origNode, path, replacement)); + } + return document; + } + + public static JsonNode getValueNode(JsonNode node, String path, String replacement) { + final String nodeClass = node.getClass().getName(); + if ( ! (node instanceof ValueNode) ) die("Path "+path+" does not refer to a value (it is a "+ nodeClass +")"); + if (node instanceof TextNode) return new TextNode(replacement); + if (node instanceof BooleanNode) return BooleanNode.valueOf(Boolean.parseBoolean(replacement)); + if (node instanceof IntNode) return new IntNode(Integer.parseInt(replacement)); + if (node instanceof LongNode) return new LongNode(Long.parseLong(replacement)); + if (node instanceof DoubleNode) return new DoubleNode(Double.parseDouble(replacement)); + if (node instanceof DecimalNode) return new DecimalNode(big(replacement)); + if (node instanceof BigIntegerNode) return new BigIntegerNode(new BigInteger(replacement)); + return die("Path "+path+" refers to an unsupported ValueNode: "+ nodeClass); + } + + public static Object getNodeAsJava(JsonNode node, String path) { + + if (node == null || node instanceof NullNode) return null; + final String nodeClass = node.getClass().getName(); + + if (node instanceof ArrayNode) { + final Object[] array = new Object[node.size()]; + for (int i=0; i map = new HashMap<>(node.size()); + for (Iterator iter = node.fieldNames(); iter.hasNext(); ) { + final String name = iter.next(); + map.put(name, getNodeAsJava(node.get(name), path+"."+name)); + } + return map; + } + + if ( ! (node instanceof ValueNode) ) return node; // return as-is... + if (node instanceof TextNode) return node.textValue(); + if (node instanceof BooleanNode) return node.booleanValue(); + if (node instanceof IntNode) return node.intValue(); + if (node instanceof LongNode) return node.longValue(); + if (node instanceof DoubleNode) return node.doubleValue(); + if (node instanceof DecimalNode) return node.decimalValue(); + if (node instanceof BigIntegerNode) return node.bigIntegerValue(); + return die("Path "+path+" refers to an unsupported ValueNode: "+ nodeClass); + } + + public static JsonNode getValueNode(Object data) { + if (data == null) return NullNode.getInstance(); + if (data instanceof Integer) return new IntNode((Integer) data); + if (data instanceof Boolean) return BooleanNode.valueOf((Boolean) data); + if (data instanceof Long) return new LongNode((Long) data); + if (data instanceof Float) return new DoubleNode((Float) data); + if (data instanceof Double) return new DoubleNode((Double) data); + if (data instanceof BigDecimal) return new DecimalNode((BigDecimal) data); + if (data instanceof BigInteger) return new BigIntegerNode((BigInteger) data); + return die("Cannot create value node from: "+data+" (type "+data.getClass().getName()+")"); + } + + public static JsonNode toNode (File f) { return fromJsonOrDie(FileUtil.toStringOrDie(f), JsonNode.class); } + + // adapted from: https://stackoverflow.com/a/11459962/1251543 + public static JsonNode mergeNodes(JsonNode mainNode, JsonNode updateNode) { + + final Iterator fieldNames = updateNode.fieldNames(); + while (fieldNames.hasNext()) { + final String fieldName = fieldNames.next(); + final JsonNode jsonNode = mainNode.get(fieldName); + // if field exists and is an embedded object + if (jsonNode != null && jsonNode.isObject()) { + mergeNodes(jsonNode, updateNode.get(fieldName)); + } else { + if (mainNode instanceof ObjectNode) { + // Overwrite field + final JsonNode value = updateNode.get(fieldName); + ((ObjectNode) mainNode).set(fieldName, value); + } + } + } + + return mainNode; + } + + public static String mergeJsonOrDie(String json, String request) { + try { + return mergeJson(json, request); + } catch (Exception e) { + return die("mergeJsonOrDie: "+e, e); + } + } + public static String mergeJson(String json, String request) throws Exception { + return mergeJson(json, fromJson(request, JsonNode.class)); + } + + public static String mergeJson(String json, Object request) throws Exception { + return json(mergeJsonNodes(json, request)); + } + + public static JsonNode mergeJsonNodes(String json, Object request) throws Exception { + if (request != null) { + if (json != null) { + final JsonNode current = fromJson(json, JsonNode.class); + final JsonNode update; + if (request instanceof JsonNode) { + update = (JsonNode) request; + } else { + update = PUBLIC_MAPPER.valueToTree(request); + } + mergeNodes(current, update); + return current; + } else { + return PUBLIC_MAPPER.valueToTree(request); + } + } + return json(json, JsonNode.class); + } + + public static JsonNode mergeJsonNodesOrDie(String json, Object request) { + try { + return mergeJsonNodes(json, request); + } catch (Exception e) { + return die("mergeJsonNodesOrDie: "+e, e); + } + } + + public static String mergeJsonOrDie(String json, Object request) { + try { + return mergeJson(json, request); + } catch (Exception e) { + return die("mergeJsonOrDie: "+e, e); + } + } + + public static JsonStringEncoder getJsonStringEncoder() { return BufferRecyclers.getJsonStringEncoder(); } + +} diff --git a/src/main/java/org/cobbzilla/util/json/main/JsonEditor.java b/src/main/java/org/cobbzilla/util/json/main/JsonEditor.java new file mode 100644 index 0000000..1c7026a --- /dev/null +++ b/src/main/java/org/cobbzilla/util/json/main/JsonEditor.java @@ -0,0 +1,37 @@ +package org.cobbzilla.util.json.main; + +import org.cobbzilla.util.io.FileUtil; +import org.cobbzilla.util.json.JsonEdit; +import org.cobbzilla.util.json.JsonEditOperation; +import org.cobbzilla.util.main.BaseMain; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +public class JsonEditor extends BaseMain { + + public static void main(String[] args) throws Exception { main(JsonEditor.class, args); } + + public void run() throws Exception { + final JsonEditorOptions options = getOptions(); + JsonEdit edit = new JsonEdit() + .setJsonData(options.getInputJson()) + .addOperation(new JsonEditOperation() + .setType(options.getOperationType()) + .setPath(options.getPath()) + .setJson(options.getValue())); + + final String json = edit.edit(); + + if (options.hasOutfile()) { + FileUtil.toFile(options.getOutfile(), json); + } else { + if (empty(json)) { + System.exit(1); + } else { + System.out.print(json); + } + } + System.exit(0); + } + +} \ No newline at end of file diff --git a/src/main/java/org/cobbzilla/util/json/main/JsonEditorOptions.java b/src/main/java/org/cobbzilla/util/json/main/JsonEditorOptions.java new file mode 100644 index 0000000..0be3f05 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/json/main/JsonEditorOptions.java @@ -0,0 +1,51 @@ +package org.cobbzilla.util.json.main; + +import lombok.Getter; +import lombok.Setter; +import org.cobbzilla.util.json.JsonEditOperationType; +import org.cobbzilla.util.main.BaseMainOptions; +import org.kohsuke.args4j.Option; + +import java.io.File; + +import static org.cobbzilla.util.io.StreamUtil.toStringOrDie; + +public class JsonEditorOptions extends BaseMainOptions { + + public static final String USAGE_CONFIG_FILE = "The JSON file to source. Default is standard input."; + public static final String OPT_CONFIG_FILE = "-f"; + public static final String LONGOPT_CONFIG_FILE = "--file"; + @Option(name=OPT_CONFIG_FILE, aliases=LONGOPT_CONFIG_FILE, usage=USAGE_CONFIG_FILE) + @Getter @Setter private File jsonFile; + + public String getInputJson() { return toStringOrDie(inStream(jsonFile)); } + + public static final String USAGE_OPERATION = "The operation to perform."; + public static final String OPT_OPERATION = "-o"; + public static final String LONGOPT_OPERATION = "--operation"; + @Option(name=OPT_OPERATION, aliases=LONGOPT_OPERATION, usage=USAGE_OPERATION) + @Getter @Setter private JsonEditOperationType operationType = JsonEditOperationType.read; + + public static final String USAGE_PATH = "The path to the JSON node where the append, replace or sort will take place. " + + "Default is root node for append or sort operations. For replace, you must specify a path. " + + "For sort operations, path must be an array."; + public static final String OPT_PATH = "-p"; + public static final String LONGOPT_PATH = "--path"; + @Option(name=OPT_PATH, aliases=LONGOPT_PATH, usage=USAGE_PATH) + @Getter @Setter private String path; + + public static final String USAGE_VALUE = "The JSON data to append or update, or the field path to sort on. Required for write and sort operations."; + public static final String OPT_VALUE = "-v"; + public static final String LONGOPT_VALUE = "--value"; + @Option(name=OPT_VALUE, aliases=LONGOPT_VALUE, usage=USAGE_VALUE) + @Getter @Setter private String value; + + public static final String USAGE_OUTPUT = "The output file. Default is standard output."; + public static final String OPT_OUTPUT = "-w"; + public static final String LONGOPT_OUTPUT = "--outfile"; + @Option(name=OPT_OUTPUT, aliases=LONGOPT_OUTPUT, usage=USAGE_OUTPUT) + @Getter @Setter private File outfile; + + public boolean hasOutfile () { return outfile != null; } + +} diff --git a/src/main/java/org/cobbzilla/util/main/BaseMain.java b/src/main/java/org/cobbzilla/util/main/BaseMain.java new file mode 100644 index 0000000..b8958b0 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/main/BaseMain.java @@ -0,0 +1,111 @@ +package org.cobbzilla.util.main; + +import lombok.AccessLevel; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.daemon.ZillaRuntime; +import org.kohsuke.args4j.CmdLineException; +import org.kohsuke.args4j.CmdLineParser; + +import static org.cobbzilla.util.daemon.ZillaRuntime.background; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.reflect.ReflectionUtil.getFirstTypeParam; +import static org.cobbzilla.util.reflect.ReflectionUtil.instantiate; + +@Slf4j +public abstract class BaseMain { + + @Getter private final OPT options = initOptions(); + protected OPT initOptions() { return instantiate(getFirstTypeParam(getClass())); } + + @Getter(value=AccessLevel.PROTECTED) private final CmdLineParser parser = new CmdLineParser(getOptions()); + + protected abstract void run() throws Exception; + + public void runOrDie () { try { run(); } catch (Exception e) { die("runOrDie: "+e, e); } } + + public Thread runInBackground () { return background(this::runOrDie); } + + @Getter private String[] args; + public void setArgs(String[] args) throws CmdLineException { + this.args = args; + try { + parser.parseArgument(args); + if (options.isHelp()) { + showHelpAndExit(); + } + } catch (Exception e) { + showHelpAndExit(e); + } + } + + protected void preRun() {} + protected void postRun() {} + + public static void main(Class clazz, String[] args) { + BaseMain m = null; + int returnValue = 0; + try { + m = clazz.getDeclaredConstructor().newInstance(); + m.setArgs(args); + m.preRun(); + m.run(); + m.postRun(); + + } catch (Exception e) { + if (m == null || m.getOptions() == null || m.getOptions().isVerboseFatalErrors()) { + final String msg = "Unexpected error: " + e + (e.getCause() != null ? " (caused by " + e.getCause() + ")" : ""); + log.error(msg, e); + ZillaRuntime.die("Unexpected error: " + e); + } else { + final String msg = e.getClass().getSimpleName() + (e.getMessage() != null ? ": " + e.getMessage() : ""); + log.error(msg); + } + returnValue = -1; + } finally { + if (m != null) m.cleanup(); + } + System.exit(returnValue); + } + + public void cleanup () {} + + public void showHelpAndExit() { + parser.printUsage(System.out); + System.exit(0); + } + + public void showHelpAndExit(String error) { showHelpAndExit(new IllegalArgumentException(error)); } + + public static final String ERR_LINE = "\n--------------------------------------------------------------------------------\n"; + + public void showHelpAndExit(Exception e) { + parser.printUsage(System.err); + if ((e instanceof CmdLineException) && !empty(e.getMessage())) { + err(ERR_LINE + " >>> " + e.getMessage() + ERR_LINE); + } + System.exit(1); + } + + public static void out(String message) { System.out.println(message); } + + public static void err (String message) { System.err.println(message); } + + public T die (String message) { + if (options.isVerboseFatalErrors()) { + log.error(message); + } + err(message); + System.exit(1); + return null; + } + + public T die (String message, Exception e) { + if (options.isVerboseFatalErrors()) { + log.error(message, e); + } + err(message + ": " + e.getClass().getName() + (!empty(e.getMessage()) ? ": "+e.getMessage(): "")); + System.exit(1); + return null; + } +} diff --git a/src/main/java/org/cobbzilla/util/main/BaseMainOptions.java b/src/main/java/org/cobbzilla/util/main/BaseMainOptions.java new file mode 100644 index 0000000..3eb1ec1 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/main/BaseMainOptions.java @@ -0,0 +1,70 @@ +package org.cobbzilla.util.main; + +import lombok.Getter; +import lombok.Setter; +import org.kohsuke.args4j.Option; + +import java.io.*; +import java.lang.reflect.Field; + +import static org.cobbzilla.util.daemon.ZillaRuntime.*; + +public class BaseMainOptions { + + public static final String USAGE_HELP = "Show help for this command"; + public static final String OPT_HELP = "-h"; + public static final String LONGOPT_HELP = "--help"; + @Option(name=OPT_HELP, aliases=LONGOPT_HELP, usage=USAGE_HELP) + @Getter @Setter private boolean help; + + public static final String USAGE_VERBOSE_FATAL_ERRORS = "Verbose fatal errors"; + public static final String OPT_VERBOSE_FATAL_ERRORS = "-z"; + public static final String LONGOPT_VERBOSE_FATAL_ERRORS = "--verbose-fatal-errors"; + @Option(name=OPT_VERBOSE_FATAL_ERRORS, aliases=LONGOPT_VERBOSE_FATAL_ERRORS, usage=USAGE_VERBOSE_FATAL_ERRORS) + @Getter @Setter private boolean verboseFatalErrors = false; + + public void out(String s) { System.out.println(s); } + public void err(String s) { System.err.println(s); } + + public static InputStream inStream (File file) { + try { return file != null ? new FileInputStream(file) : System.in; } catch (Exception e) { + return die("inStream: "+e, e); + } + } + public static OutputStream outStream (File file) { + try { return file != null ? new FileOutputStream(file) : System.out; } catch (Exception e) { + return die("outStream: "+e, e); + } + } + + public static BufferedReader reader (File file) { + try { return file != null ? new BufferedReader(new FileReader(file)) : stdin(); } catch (Exception e) { + return die("reader: "+e, e); + } + } + public static BufferedWriter writer (File file) { + try { return new BufferedWriter(file != null ? new FileWriter(file) : stdout()); } catch (Exception e) { + return die("writer: "+e, e); + } + } + + public void required(String field) { + try { + final Field optField = getClass().getField("OPT_"+field); + final Field longOptField = getClass().getField("LONGOPT_"+field); + err("Missing option: "+optField.get(null)+"/"+longOptField.get(null)); + } catch (Exception e) { + die("No such field: "+field+": "+e, e); + } + } + + public void requiredAndDie(String field) { + try { + final Field optField = getClass().getField("OPT_"+field); + final Field longOptField = getClass().getField("LONGOPT_"+field); + die("Missing option: "+optField.get(null)+"/"+longOptField.get(null)); + } catch (Exception e) { + die("No such field: "+field+": "+e, e); + } + } +} diff --git a/src/main/java/org/cobbzilla/util/main/IndexMain.java b/src/main/java/org/cobbzilla/util/main/IndexMain.java new file mode 100644 index 0000000..badf5d9 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/main/IndexMain.java @@ -0,0 +1,63 @@ +package org.cobbzilla.util.main; + +import com.google.common.collect.ImmutableMap; +import org.cobbzilla.util.collection.ArrayUtil; +import org.cobbzilla.util.io.main.FilesystemWatcherMain; +import org.cobbzilla.util.io.main.JarTrimmerMain; +import org.cobbzilla.util.json.main.JsonEditor; +import org.cobbzilla.util.string.StringUtil; +import org.slf4j.bridge.SLF4JBridgeHandler; + +import java.lang.reflect.Method; +import java.util.Map; + +public class IndexMain { + + public static final Map> handlers + = ImmutableMap.>builder() + .put("json", JsonEditor.class) + .put("fswatch", FilesystemWatcherMain.class) + .put("trim-jar", JarTrimmerMain.class) + .build(); + public Map> getHandlers() { return handlers; } + + public static void main (String[] args) { main(IndexMain.class, args); } + + protected static void main(Class clazz, String[] args) { + + // redirect JUL -> logback using slf4j + SLF4JBridgeHandler.removeHandlersForRootLogger(); + SLF4JBridgeHandler.install(); + + if (args.length == 0) die("No command given. Use one of these: " + StringUtil.toString(handlers.keySet(), " ")); + + // find the command + final Class handler = handlers.get(args[0]); + if (handler == null) die("Unrecognized command: "+args[0]); + + // find the main method + final Method mainMethod; + try { + mainMethod = handler.getMethod("main", String[].class); + } catch (Exception e) { + die("Error loading main method: "+e); + return; + } + if (mainMethod == null) die("No main method found for "+handler.getName()); + + // strip first arg (command name) and call main + final String[] newArgs = ArrayUtil.remove(args, 0); + try { + mainMethod.invoke(null, (Object) newArgs); + } catch (Exception e) { + die("Error running main method: "+e); + } + + } + + protected static void die(String msg) { + System.err.println(msg); + System.exit(1); + } + +} diff --git a/src/main/java/org/cobbzilla/util/math/Cardinal.java b/src/main/java/org/cobbzilla/util/math/Cardinal.java new file mode 100644 index 0000000..cc72931 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/math/Cardinal.java @@ -0,0 +1,39 @@ +package org.cobbzilla.util.math; + +import com.fasterxml.jackson.annotation.JsonCreator; +import lombok.Getter; + +public enum Cardinal { + + north (1, "N", "north"), + east (1, "E", "east"), + south (-1, "S", "south"), + west (-1, "W", "west"); + + @Getter private final int direction; + @Getter private final String[] allAliases; + + Cardinal(int direction, String... allAliases) { + this.direction = direction; + this.allAliases = allAliases; + } + + @JsonCreator public static Cardinal create (String val) { + for (Cardinal c : values()) { + for (String a : c.allAliases) { + if (a.equalsIgnoreCase(val)) return c; + } + } + return null; + } + + @Override public String toString () { return allAliases[0]; } + + public static boolean isCardinal(String val) { + try { + return create(val.toLowerCase()) != null; + } catch (Exception e) { + return false; + } + } +} diff --git a/src/main/java/org/cobbzilla/util/math/CumipmtCalculator.java b/src/main/java/org/cobbzilla/util/math/CumipmtCalculator.java new file mode 100644 index 0000000..e17b957 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/math/CumipmtCalculator.java @@ -0,0 +1,41 @@ +package org.cobbzilla.util.math; + +import static java.lang.Math.pow; + +public class CumipmtCalculator { + + public static double cumipmt(double ratePerPeriod, int paymentsCount, double financed, int startPeriod, + int endPeriod, boolean isDueOnPeriodEnd) { + if (startPeriod < 1 || endPeriod < startPeriod || ratePerPeriod <= 0 || endPeriod > paymentsCount || + paymentsCount <= 0 || financed <= 0) throw new IllegalArgumentException(); + + double payment = 0; + + if (startPeriod == 1) { + if (isDueOnPeriodEnd) payment = -financed; + startPeriod++; + } + + double rmz = -financed * ratePerPeriod / (1 - 1 / pow(1 + ratePerPeriod, paymentsCount)); + if (!isDueOnPeriodEnd) rmz /= 1 + ratePerPeriod; + + for (int i=startPeriod; i<=endPeriod; i++) { + payment += getPeriodPaymentAddOn(ratePerPeriod, i, rmz, financed, isDueOnPeriodEnd); + } + + return payment * ratePerPeriod; + } + + private static double getPeriodPaymentAddOn(double ratePerPeriod, double periodIndex, double rmz, double financed, + boolean isDueOnPeriodEnd) { + double term = pow(1 + ratePerPeriod, periodIndex - (isDueOnPeriodEnd ? 1 : 2)); + + double addOn = rmz * (term - 1) / ratePerPeriod; + if (!isDueOnPeriodEnd) addOn *= 1 + ratePerPeriod; + + double res = -(financed * term + addOn); + if (!isDueOnPeriodEnd) res -= rmz; + + return res; + } +} diff --git a/src/main/java/org/cobbzilla/util/math/Haversine.java b/src/main/java/org/cobbzilla/util/math/Haversine.java new file mode 100644 index 0000000..fefc73a --- /dev/null +++ b/src/main/java/org/cobbzilla/util/math/Haversine.java @@ -0,0 +1,39 @@ +package org.cobbzilla.util.math; + +public class Haversine { + + public static double distance(double lat1, double lat2, double lon1, double lon2) { + return distance(lat1, lat2, lon1, lon2, 0, 0); + } + + // adapted from https://stackoverflow.com/a/16794680/1251543 + /** + * Calculate distance between two points in latitude and longitude taking + * into account height difference. If you are not interested in height + * difference pass 0.0. Uses Haversine method as its base. + * + * lat1, lon1 Start point lat2, lon2 End point el1 Start altitude in meters + * el2 End altitude in meters + * @returns Distance in Meters + */ + public static double distance(double lat1, double lat2, double lon1, + double lon2, double el1, double el2) { + + final int R = 6371; // Radius of the earth + + double latDistance = Math.toRadians(lat2 - lat1); + double lonDistance = Math.toRadians(lon2 - lon1); + double a = Math.sin(latDistance / 2) * Math.sin(latDistance / 2) + + Math.cos(Math.toRadians(lat1)) * Math.cos(Math.toRadians(lat2)) + * Math.sin(lonDistance / 2) * Math.sin(lonDistance / 2); + double c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)); + double distance = R * c * 1000; // convert to meters + + double height = el1 - el2; + + distance = Math.pow(distance, 2) + Math.pow(height, 2); + + return Math.sqrt(distance); + } + +} diff --git a/src/main/java/org/cobbzilla/util/network/NetworkInterfaceType.java b/src/main/java/org/cobbzilla/util/network/NetworkInterfaceType.java new file mode 100644 index 0000000..de5a874 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/network/NetworkInterfaceType.java @@ -0,0 +1,11 @@ +package org.cobbzilla.util.network; + +import com.fasterxml.jackson.annotation.JsonCreator; + +public enum NetworkInterfaceType { + + local, world, vpn, vpn2, custom; + + @JsonCreator public static NetworkInterfaceType create (String v) { return valueOf(v.toLowerCase()); } + +} diff --git a/src/main/java/org/cobbzilla/util/network/NetworkPort.java b/src/main/java/org/cobbzilla/util/network/NetworkPort.java new file mode 100644 index 0000000..9c8a840 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/network/NetworkPort.java @@ -0,0 +1,14 @@ +package org.cobbzilla.util.network; + +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; + +@Accessors(chain=true) +public class NetworkPort { + + @Getter @Setter Integer port; + @Getter @Setter TransportProtocol protocol = TransportProtocol.tcp; + @Getter @Setter NetworkInterfaceType iface = NetworkInterfaceType.world; + +} diff --git a/src/main/java/org/cobbzilla/util/network/NetworkUtil.java b/src/main/java/org/cobbzilla/util/network/NetworkUtil.java new file mode 100644 index 0000000..be46e6c --- /dev/null +++ b/src/main/java/org/cobbzilla/util/network/NetworkUtil.java @@ -0,0 +1,185 @@ +package org.cobbzilla.util.network; + +import com.sun.jna.Platform; +import lombok.extern.slf4j.Slf4j; + +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.util.Enumeration; +import java.util.HashSet; +import java.util.Set; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.string.ValidationRegexes.IPv4_PATTERN; + +@Slf4j +public class NetworkUtil { + + public static final String IPv4_ALL_ADDRS = "0.0.0.0"; + public static final String IPv4_LOCALHOST = "127.0.0.1"; + + public static boolean isLocalIpv4(String addr) { + if (empty(addr)) return false; + if (addr.startsWith("/")) addr = addr.substring(1); + if (!IPv4_PATTERN.matcher(addr).matches()) return false; + if (addr.startsWith("127.")) return true; + return false; + } + + public static boolean isLocalHost(String host) { + if (isLocalIpv4(host)) return true; + try { + return isLocalIpv4(InetAddress.getByName(host).getHostAddress()); + } catch (Exception e) { + log.warn("isLocalHost("+host+"): "+e); + return false; + } + } + + public static boolean isPublicIpv4(String addr) { + if (empty(addr)) return false; + if (addr.startsWith("/")) addr = addr.substring(1); + if (!IPv4_PATTERN.matcher(addr).matches()) return false; + try { + final InetAddress address = InetAddress.getByName(addr); + if (address.isSiteLocalAddress() || address.isLoopbackAddress() || address.isLinkLocalAddress()) return false; + return !isLocalIpv4(addr) && !isPrivateIp4(addr); + } catch (Exception e) { + log.warn("isPublicIpv4: "+e); + return false; + } + } + + public static boolean isPrivateIp4(String addr) { + if (addr.startsWith("10.")) return true; + if (addr.startsWith("192.168.")) return true; + if (addr.startsWith("172.")) { + final String remainder = addr.substring("172.".length()); + final int secondDot = remainder.indexOf("."); + final String secondPart = remainder.substring(0, secondDot); + final int octet = Integer.parseInt(secondPart); + return octet >= 16 && octet <= 31; + } + return false; + } + + public static String getEthernetIpv4(NetworkInterface iface) { + if (iface == null) return null; + if (!iface.getName().startsWith(getEthernetInterfacePrefix())) return null; + final Enumeration addrs = iface.getInetAddresses(); + while (addrs.hasMoreElements()) { + String addr = addrs.nextElement().toString(); + if (addr.startsWith("/")) addr = addr.substring(1); + if (!IPv4_PATTERN.matcher(addr).matches()) continue; + return addr; + } + return null; + } + + protected static String getEthernetInterfacePrefix() { + if (Platform.isWindows() || Platform.isLinux()) return "eth"; + if (Platform.isMac()) return "en"; + return die("getEthernetInterfacePrefix: unknown platform "+System.getProperty("os.name")); + } + + protected static String getLocalInterfacePrefix() { + return "lo"; + } + + public static String getLocalhostIpv4 () { + try { + final Enumeration interfaces = NetworkInterface.getNetworkInterfaces(); + while (interfaces.hasMoreElements()) { + final NetworkInterface i = interfaces.nextElement(); + if (i.getName().startsWith(getLocalInterfacePrefix())) { + final Enumeration addrs = i.getInetAddresses(); + while (addrs.hasMoreElements()) { + String addr = addrs.nextElement().toString(); + if (addr.startsWith("/")) addr = addr.substring(1); + if (isLocalIpv4(addr)) return addr; + } + } + } + return die("getLocalhostIpv4: no local 127.x.x.x address found"); + + } catch (Exception e) { + return die("getLocalhostIpv4: "+e, e); + } + } + + public static Set configuredIps() { + final Set ips = new HashSet<>(); + try { + final Enumeration interfaces = NetworkInterface.getNetworkInterfaces(); + while (interfaces.hasMoreElements()) { + final NetworkInterface i = interfaces.nextElement(); + final Enumeration addrs = i.getInetAddresses(); + while (addrs.hasMoreElements()) { + String ip = addrs.nextElement().toString(); + if (ip.startsWith("/")) ip = ip.substring(1); + ips.add(ip); + } + } + return ips; + } catch (Exception e) { + return die("configuredIps: "+e, e); + } + } + + public static String getFirstPublicIpv4() { + try { + final Enumeration interfaces = NetworkInterface.getNetworkInterfaces(); + while (interfaces.hasMoreElements()) { + final NetworkInterface i = interfaces.nextElement(); + final Enumeration addrs = i.getInetAddresses(); + while (addrs.hasMoreElements()) { + final String addr = addrs.nextElement().toString(); + if (isPublicIpv4(addr)) { + return addr.substring(1); + } + } + } + log.warn("getFirstPublicIpv4: no public IPv4 address found"); + return null; + + } catch (Exception e) { + return die("getFirstPublicIpv4: "+e, e); + } + } + + public static String getFirstEthernetIpv4() { + try { + final Enumeration interfaces = NetworkInterface.getNetworkInterfaces(); + while (interfaces.hasMoreElements()) { + final NetworkInterface i = interfaces.nextElement(); + final String addr = getEthernetIpv4(i); + if (!empty(addr)) return addr; + } + log.warn("getFirstEthernetIpv4: no ethernet IPv4 address found"); + return null; + + } catch (Exception e) { + return die("getFirstPublicIpv4: "+e, e); + } + } + + public static String getInAddrArpa(String ip) { + final String[] parts = ip.split("\\."); + return new StringBuilder() + .append(parts[3]).append('.') + .append(parts[2]).append('.') + .append(parts[1]).append('.') + .append(parts[0]).append(".in-addr.arpa") + .toString(); + } + + public static boolean ipEquals(String addr1, String addr2) { + try { + return InetAddress.getByName(addr1).equals(InetAddress.getByName(addr2)); + } catch (Exception e) { + log.warn("ipEquals: "+e); + return false; + } + } +} diff --git a/src/main/java/org/cobbzilla/util/network/PortPicker.java b/src/main/java/org/cobbzilla/util/network/PortPicker.java new file mode 100644 index 0000000..22cbe6b --- /dev/null +++ b/src/main/java/org/cobbzilla/util/network/PortPicker.java @@ -0,0 +1,24 @@ +package org.cobbzilla.util.network; + +import java.io.IOException; +import java.net.ServerSocket; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; + +public class PortPicker { + + public static int pick() throws IOException { + try (ServerSocket s = new ServerSocket(0)) { + return s.getLocalPort(); + } + } + + public static int pickOrDie() { + try { + return pick(); + } catch (IOException e) { + return die("Error picking port: "+e, e); + } + } + +} \ No newline at end of file diff --git a/src/main/java/org/cobbzilla/util/network/TransportProtocol.java b/src/main/java/org/cobbzilla/util/network/TransportProtocol.java new file mode 100644 index 0000000..1276b1a --- /dev/null +++ b/src/main/java/org/cobbzilla/util/network/TransportProtocol.java @@ -0,0 +1,11 @@ +package org.cobbzilla.util.network; + +import com.fasterxml.jackson.annotation.JsonCreator; + +public enum TransportProtocol { + + icmp, udp, tcp; + + @JsonCreator public static TransportProtocol create (String v) { return valueOf(v.toLowerCase()); } + +} diff --git a/src/main/java/org/cobbzilla/util/reflect/ClassReLoader.java b/src/main/java/org/cobbzilla/util/reflect/ClassReLoader.java new file mode 100644 index 0000000..b56fbc3 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/reflect/ClassReLoader.java @@ -0,0 +1,35 @@ +package org.cobbzilla.util.reflect; + +import java.net.URLClassLoader; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +import static java.lang.Class.forName; + +// adapted from https://stackoverflow.com/a/9192126/1251543 +public class ClassReLoader extends URLClassLoader { + + private Set reload = new HashSet<>(); + public void doReloadFor(String classOrPackage) { reload.add(classOrPackage); } + + public ClassReLoader(Collection toReload) { + super(((URLClassLoader) getSystemClassLoader()).getURLs()); + reload.addAll(toReload); + } + + @Override public Class loadClass(String name) throws ClassNotFoundException { + if (name.startsWith("java.")) return forName(name); + if (!reload.contains(name)) { + String find = name; + while (find.contains(".")) { + find = find.substring(0, find.lastIndexOf(".")); + if (reload.contains(find)) { + return super.loadClass(name); + } + } + return forName(name); + } + return super.loadClass(name); + } +} diff --git a/src/main/java/org/cobbzilla/util/reflect/FieldComparator.java b/src/main/java/org/cobbzilla/util/reflect/FieldComparator.java new file mode 100644 index 0000000..e4e9309 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/reflect/FieldComparator.java @@ -0,0 +1,22 @@ +package org.cobbzilla.util.reflect; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.util.Comparator; + +@AllArgsConstructor +public class FieldComparator> implements Comparator { + + @Getter private final String field; + @Getter private final boolean reverse = false; + + @Override public int compare(T o1, T o2) { + final F v1 = (F) ReflectionUtil.get(o1, field); + final F v2 = (F) ReflectionUtil.get(o2, field); + return reverse + ? (v1 == null ? (v2 == null ? 0 : 1) : (v2 == null ? 1 : v2.compareTo(v1))) + : (v1 == null ? (v2 == null ? 0 : -1) : (v2 == null ? -1 : v1.compareTo(v2))); + } + +} diff --git a/src/main/java/org/cobbzilla/util/reflect/Immutable.java b/src/main/java/org/cobbzilla/util/reflect/Immutable.java new file mode 100644 index 0000000..b075eba --- /dev/null +++ b/src/main/java/org/cobbzilla/util/reflect/Immutable.java @@ -0,0 +1,34 @@ +package org.cobbzilla.util.reflect; + +import lombok.AllArgsConstructor; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; + +@AllArgsConstructor +public class Immutable implements InvocationHandler { + + private final T obj; + + public Object invoke(Object proxy, Method m, Object[] args) throws Throwable { + + final String mName = m.getName(); + if (!(mName.startsWith("get") + || mName.startsWith("is") + || m.getParameterTypes().length > 0 + || Void.class.isAssignableFrom(m.getReturnType()) + )) die("invoke("+obj.getClass().getSimpleName()+"."+mName+"): not a zero-arg getter or returns void: "+mName); + + return m.invoke(obj, args); + } + + public static T wrap(T thing) { + final ClassLoader loader = thing.getClass().getClassLoader(); + final Class[] classes = thing.getClass().getInterfaces(); + return (T) Proxy.newProxyInstance(loader, classes, new Immutable(thing)); + } + +} diff --git a/src/main/java/org/cobbzilla/util/reflect/NoSetters.java b/src/main/java/org/cobbzilla/util/reflect/NoSetters.java new file mode 100644 index 0000000..4877a02 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/reflect/NoSetters.java @@ -0,0 +1,23 @@ +package org.cobbzilla.util.reflect; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; + +import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported; + +public class NoSetters { + + public static T wrap(T thing) { + return (T) Proxy.newProxyInstance(thing.getClass().getClassLoader(), new Class[]{thing.getClass()}, NoSettersInvocationHandler.instance); + } + + private static class NoSettersInvocationHandler implements InvocationHandler { + public static InvocationHandler instance; + @Override public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + if (method.getName().startsWith("set")) return notSupported("immutable object: " + proxy + ", cannot call " + method.getName()); + return method.invoke(proxy, args); + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/reflect/ObjectFactory.java b/src/main/java/org/cobbzilla/util/reflect/ObjectFactory.java new file mode 100644 index 0000000..7ad24f4 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/reflect/ObjectFactory.java @@ -0,0 +1,10 @@ +package org.cobbzilla.util.reflect; + +import java.util.Map; + +public interface ObjectFactory { + + T create (); + T create (Map ctx); + +} diff --git a/src/main/java/org/cobbzilla/util/reflect/OnlyGetters.java b/src/main/java/org/cobbzilla/util/reflect/OnlyGetters.java new file mode 100644 index 0000000..974124d --- /dev/null +++ b/src/main/java/org/cobbzilla/util/reflect/OnlyGetters.java @@ -0,0 +1,28 @@ +package org.cobbzilla.util.reflect; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; + +import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported; + +public class OnlyGetters { + + public static T wrap(T thing) { + return (T) Proxy.newProxyInstance(thing.getClass().getClassLoader(), new Class[]{thing.getClass()}, OnlyGettersInvocationHandler.instance); + } + + private static class OnlyGettersInvocationHandler implements InvocationHandler { + public static InvocationHandler instance; + @Override public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + final String name = method.getName(); + if ((args == null || args.length == 0) + && (name.startsWith("get") || name.startsWith("is") + && !method.getReturnType().equals(Void.class))) { + return notSupported("immutable object: " + proxy + ", cannot call " + name); + } + return method.invoke(proxy, args); + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/reflect/PoisonProxy.java b/src/main/java/org/cobbzilla/util/reflect/PoisonProxy.java new file mode 100644 index 0000000..25d6368 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/reflect/PoisonProxy.java @@ -0,0 +1,44 @@ +package org.cobbzilla.util.reflect; + +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; + +import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported; + +public class PoisonProxy { + + public interface PoisonProxyThrow { Throwable getThrowable(Object proxy, Method method, Object[] args); } + + public static T wrap(Class clazz) { return wrap(clazz, null); } + + public static T wrap(Class clazz, PoisonProxyThrow thrower) { return wrap(new Class[]{clazz}, thrower); } + + /** + * Create a proxy object for a class where calling any methods on the object will result in it throwing an exception. + * @param clazzes The classes to create a proxy for + * @param thrower An object implementing the PoisonProxyThrow interface, which produces objects to throw + * @param The class to create a proxy for + * @return A proxy to the class that will throw an exception if any methods are called on it + */ + public static T wrap(Class[] clazzes, PoisonProxyThrow thrower) { + return (T) Proxy.newProxyInstance(clazzes[0].getClassLoader(), clazzes, thrower == null ? PoisonedInvocationHandler.instance : new PoisonedInvocationHandler(thrower)); + } + + @NoArgsConstructor @AllArgsConstructor + private static class PoisonedInvocationHandler implements InvocationHandler { + public static PoisonedInvocationHandler instance = new PoisonedInvocationHandler(); + private PoisonProxyThrow thrower = null; + @Override public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + if (thrower == null) { + return notSupported("method not supported by poisonProxy: " + method.getName() + " (in fact, NO methods will work on this object)"); + } else { + throw thrower.getThrowable(proxy, method, args); + } + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/reflect/ReflectionUtil.java b/src/main/java/org/cobbzilla/util/reflect/ReflectionUtil.java new file mode 100644 index 0000000..7028aed --- /dev/null +++ b/src/main/java/org/cobbzilla/util/reflect/ReflectionUtil.java @@ -0,0 +1,1075 @@ +package org.cobbzilla.util.reflect; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.beanutils.MethodUtils; +import org.apache.commons.collections.Transformer; +import org.apache.commons.lang3.ArrayUtils; + +import java.io.Closeable; +import java.lang.annotation.Annotation; +import java.lang.reflect.*; +import java.math.BigDecimal; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +import static com.google.common.base.Preconditions.checkNotNull; +import static org.cobbzilla.util.collection.ArrayUtil.arrayToString; +import static org.cobbzilla.util.daemon.ZillaRuntime.*; +import static org.cobbzilla.util.string.StringUtil.uncapitalize; + +/** + * Handy tools for working quickly with reflection APIs, which tend to be verbose. + */ +@Slf4j +public class ReflectionUtil { + + public static final Class[] EMPTY_CLASS_ARRAY = new Class[0]; + public static final Class[] SINGLE_STRING_ARG = {String.class}; + + public static Boolean toBoolean(Object object) { + if (object == null) return null; + if (object instanceof Boolean) return (Boolean) object; + if (object instanceof String) return Boolean.valueOf(object.toString()); + return null; + } + + public static Boolean toBoolean(Object object, String field, boolean defaultValue) { + final Boolean val = toBoolean(get(object, field)); + return val == null ? defaultValue : val; + } + + public static Long toLong(Object object) { + if (object == null) return null; + if (object instanceof Number) return ((Number) object).longValue(); + if (object instanceof String) return Long.valueOf(object.toString()); + return null; + } + + public static Integer toInteger(Object object) { + if (object == null) return null; + if (object instanceof Number) return ((Number) object).intValue(); + if (object instanceof String) return Integer.valueOf(object.toString()); + return null; + } + + public static Integer toIntegerOrNull(Object object) { + if (object == null) return null; + if (object instanceof Number) return ((Number) object).intValue(); + if (object instanceof String) { + try { + return Integer.valueOf(object.toString()); + } catch (Exception e) { + log.info("toIntegerOrNull("+object+"): "+e); + return null; + } + } + return null; + } + + public static Short toShort(Object object) { + if (object == null) return null; + if (object instanceof Number) return ((Number) object).shortValue(); + if (object instanceof String) return Short.valueOf(object.toString()); + return null; + } + + public static Float toFloat(Object object) { + if (object == null) return null; + if (object instanceof Number) return ((Number) object).floatValue(); + if (object instanceof String) return Float.valueOf(object.toString()); + return null; + } + + public static Double toDouble(Object object) { + if (object == null) return null; + if (object instanceof Number) return ((Number) object).doubleValue(); + if (object instanceof String) return Double.valueOf(object.toString()); + return null; + } + + public static BigDecimal toBigDecimal(Object object) { + if (object == null) return null; + if (object instanceof Double) return big((Double) object); + if (object instanceof Float) return big((Float) object); + if (object instanceof Number) return big(((Number) object).longValue()); + if (object instanceof String) return big(object.toString()); + return null; + } + + /** + * Do a Class.forName and only throw unchecked exceptions. + * @param clazz full class name. May end in [] to indicate array class + * @param The class type + * @return A Class<clazz> object + */ + public static Class forName(String clazz) { + if (empty(clazz)) return (Class) Object.class; + if (clazz.endsWith("[]")) return arrayClass(forName(clazz.substring(0, clazz.length()-2))); + try { + return (Class) Class.forName(clazz); + } catch (Exception e) { + return die("Class.forName("+clazz+") error: "+e, e); + } + } + + public static Collection forNames(String[] classNames) { + final List list = new ArrayList<>(); + if (!empty(classNames)) for (String c : classNames) list.add(forName(c)); + return list; + } + + public static Class arrayClass (Class clazz) { return forName("[L"+clazz.getName()+";"); } + + /** + * Create an instance of a class, only throwing unchecked exceptions. The class must have a default constructor. + * @param clazz we will instantiate an object of this type + * @param The class type + * @return An Object that is an instance of Class<clazz> object + */ + public static T instantiate(Class clazz) { + try { + return clazz.getDeclaredConstructor().newInstance(); + } catch (Exception e) { + return die("Error instantiating "+clazz+": "+e, e); + } + } + + /** + * Create an instance of a class based on a class name, only throwing unchecked exceptions. The class must have a default constructor. + * @param clazz full class name + * @param The class type + * @return An Object that is an instance of Class<clazz> object + */ + public static T instantiate(String clazz) { + try { + return (T) instantiate(forName(clazz)); + } catch (Exception e) { + return die("instantiate("+clazz+"): "+e, e); + } + } + + private static final Map> enumCache = new ConcurrentHashMap<>(1000); + + /** + * Create an instance of a class using the supplied argument to a matching single-argument constructor. + * @param clazz The class to instantiate + * @param argument The object that will be passed to a matching single-argument constructor + * @param Could be anything + * @return A new instance of clazz, created using a constructor that matched argument's class. + */ + public static T instantiate(Class clazz, Object argument) { + Constructor constructor = null; + Class tryClass = argument.getClass(); + if (clazz.isPrimitive()) { + switch (clazz.getName()) { + case "boolean": return (T) Boolean.valueOf(argument.toString()); + case "byte": return (T) Byte.valueOf(argument.toString()); + case "short": return (T) Short.valueOf(argument.toString()); + case "char": return (T) Character.valueOf(empty(argument) ? 0 : argument.toString().charAt(0)); + case "int": return (T) Integer.valueOf(argument.toString()); + case "long": return (T) Long.valueOf(argument.toString()); + case "float": return (T) Float.valueOf(argument.toString()); + case "double": return (T) Double.valueOf(argument.toString()); + default: return die("instantiate: unrecognized primitive type: "+clazz.getName()); + } + } + if (clazz.isEnum()) { + return argument == null ? null : (T) enumCache + .computeIfAbsent(clazz, c -> new ConcurrentHashMap<>(50)) + .computeIfAbsent(argument, e -> { + try { + final Method valueOf = clazz.getMethod("valueOf", SINGLE_STRING_ARG); + return (Enum) valueOf.invoke(null, new Object[]{argument.toString()}); + } catch (Exception ex) { + return die("instantiate: error instantiating enum "+clazz.getName()+": "+e); + } + }); + } + while (constructor == null) { + try { + constructor = clazz.getConstructor(tryClass); + } catch (NoSuchMethodException e) { + if (tryClass.equals(Object.class)) { + // try interfaces + for (Class iface : argument.getClass().getInterfaces()) { + try { + constructor = clazz.getConstructor(iface); + } catch (NoSuchMethodException e2) { + // noop + } + } + break; + } else { + tryClass = tryClass.getSuperclass(); + } + } + } + if (constructor == null) { + die("instantiate: no constructor could be found for class "+clazz.getName()+", argument type "+argument.getClass().getName()); + } + try { + return constructor.newInstance(argument); + } catch (Exception e) { + return die("instantiate("+clazz.getName()+", "+argument+"): "+e, e); + } + } + + /** + * Create an instance of a class using the supplied argument to a matching single-argument constructor. + * @param clazz The class to instantiate + * @param arguments The objects that will be passed to a matching constructor + * @param Could be anything + * @return A new instance of clazz, created using a constructor that matched argument's class. + */ + public static T instantiate(Class clazz, Object... arguments) { + try { + for (Constructor constructor : clazz.getConstructors()) { + final Class[] cParams = constructor.getParameterTypes(); + if (cParams.length == arguments.length) { + boolean match = true; + for (int i=0; i getSimpleClass(Object argument) { + Class argClass = argument.getClass(); + final int enhancePos = argClass.getName().indexOf("$$Enhance"); + if (enhancePos != -1) { + argClass = forName(argClass.getName().substring(0, enhancePos)); + } + return argClass; + } + + public static String getSimpleClassName(Object argument) { return getSimpleClass(argument).getClass().getSimpleName(); } + + /** + * Make a copy of the object, assuming its class has a copy constructor + * @param thing The thing to copy + * @param Whatevs + * @return A copy of the object, created using the thing's copy constructor + */ + public static T copy(T thing) { return (T) instantiate(thing.getClass(), thing); } + + /** + * Mirror the object. Create a new instance and copy all fields + * @param thing The thing to copy + * @param Whatevs + * @return A mirror of the object, created using the thing's default constructor and copying all fields with 'copy' + */ + public static T mirror(T thing) { + T copy = (T) instantiate(thing.getClass()); + copy(copy, thing); + return copy; + } + + public static Object invokeStatic(Method m, Object... values) { + try { + return m.invoke(null, values); + } catch (Exception e) { + return die("invokeStatic: "+m.getClass().getSimpleName()+"."+m.getName()+"("+arrayToString(values, ", ")+"): "+e, e); + } + } + + public static Field getDeclaredField(Class clazz, String field) { + try { + return clazz.getDeclaredField(field); + } catch (NoSuchFieldException e) { + if (clazz.equals(Object.class)) { + log.info("getDeclaredField: field not found "+clazz.getName()+"/"+field); + return null; + } + } + return getDeclaredField(clazz.getSuperclass(), field); + } + + public static Field getField(Class clazz, String field) { + try { + return clazz.getField(field); + } catch (NoSuchFieldException e) { + if (clazz.equals(Object.class)) { + log.info("getField: field not found "+clazz.getName()+"/"+field); + return null; + } + } + return getDeclaredField(clazz.getSuperclass(), field); + } + + public static Method factoryMethod(Class clazz, Object value) { + // find a static method that takes the value and returns an instance of the class + for (Method m : clazz.getMethods()) { + if (m.getReturnType().equals(clazz)) { + final Class[] parameterTypes = m.getParameterTypes(); + if (parameterTypes != null && parameterTypes.length == 1 && parameterTypes[0].isAssignableFrom(value.getClass())) { + return m; + } + } + } + log.warn("factoryMethod: class "+clazz.getName()+" does not have static factory method that takes a String, returning null"); + return null; + } + + public static T callFactoryMethod(Class clazz, Object value) { + final Method m = factoryMethod(clazz, value); + return m != null ? (T) invokeStatic(m, value) : null; + } + + public static Object scrubStrings(Object thing, String[] fields) { + if (empty(thing)) return thing; + if (thing.getClass().isPrimitive() + || thing instanceof String + || thing instanceof Number + || thing instanceof Enum) return thing; + + if (thing instanceof JsonNode) { + if (thing instanceof ObjectNode) { + for (String field : fields) { + if (((ObjectNode) thing).has(field)) { + ((ObjectNode) thing).remove(field); + } + } + } else if (thing instanceof ArrayNode) { + ArrayNode arrayNode = (ArrayNode) thing; + for (int i = 0; i < arrayNode.size(); i++) { + scrubStrings(arrayNode.get(i), fields); + } + } + } else if (thing instanceof Map) { + final Map map = (Map) thing; + final Set toRemove = new HashSet(); + for (Object e : map.entrySet()) { + Map.Entry entry = (Map.Entry) e; + if (ArrayUtils.contains(fields, entry.getKey().toString())) { + toRemove.add(entry.getKey()); + } else { + scrubStrings(entry.getValue(), fields); + } + } + for (Object key : toRemove) map.remove(key); + + } else if (Object[].class.isAssignableFrom(thing.getClass())) { + if ( !((Object[]) thing)[0].getClass().isPrimitive() ) { + for (Object obj : ((Object[]) thing)) { + scrubStrings(obj, fields); + } + } + } else if (thing instanceof Collection) { + for (Object obj : ((Collection) thing)) { + scrubStrings(obj, fields); + } + } else { + for (String field : ReflectionUtil.toMap(thing).keySet()) { + final Object val = get(thing, field, null); + if (val != null) { + if (ArrayUtils.contains(fields, field)) { + setNull(thing, field, String.class); + } else { + scrubStrings(val, fields); + } + } + } + } + return thing; + } + + private enum Accessor { get, set } + + /** + * Copies fields from src to dest. Code is easier to read if this method is understdood to be like an assignment statement, dest = src + * + * We consider only 'getter' methods that meet the following criteria: + * (1) starts with "get" + * (2) takes zero arguments + * (3) has a return value + * (4) does not carry any annotation whose simple class name is "Transient" + * + * The value returned from the source getter will be copied to the destination (via setter), if a setter exists, and: + * (1) No getter exists on the destination, or (2) the destination's getter returns a different value (.equals returns false) + * + * Getters that return null values on the source object will not be copied. + * + * @param dest destination object + * @param src source object + * @param objects must share a type + * @return count of fields copied + */ + public static int copy (T dest, T src) { + return copy(dest, src, null, null); + } + + /** + * Same as copy(dest, src) but only named fields are copied + * @param dest destination object + * @param src source object + * @param fields only fields with these names will be considered for copying + * @param objects must share a type + * @return count of fields copied + */ + public static int copy (T dest, T src, String[] fields) { + int copyCount = 0; + if (fields != null) { + for (String field : fields) { + try { + final Object value = get(src, field, null); + if (value != null) { + set(dest, field, value); + copyCount++; + } + } catch (Exception e) { + log.debug("copy: field=" + field + ": " + e); + } + } + } + return copyCount; + } + + /** + * Same as copy(dest, src) but only named fields are copied + * @param dest destination object, or a Map + * @param src source object + * @param fields only fields with these names will be considered for copying + * @param exclude fields with these names will NOT be considered for copying + * @param objects must share a type + * @return count of fields copied + */ + public static int copy (T dest, T src, String[] fields, String[] exclude) { + int copyCount = 0; + final boolean isMap = dest instanceof Map; + try { + if (src instanceof Map) copyFromMap(dest, (Map) src, exclude); + + checkGetter: + for (Method getter : src.getClass().getMethods()) { + // only look for getters on the source object (methods with no arguments that have a return value) + final Class[] types = getter.getParameterTypes(); + if (types.length != 0) continue; + if (getter.getReturnType().equals(Void.class)) continue;; + + // and it must be named appropriately + final String fieldName = fieldName(getter.getName()); + if (fieldName == null || ArrayUtils.contains(exclude, fieldName)) continue; + + // if specific fields were given, it must be one of those + if (fields != null && !ArrayUtils.contains(fields, fieldName)) continue; + + // getter must not be marked @Transient + if (isIgnored(src, fieldName, getter)) continue; + + // what would the setter be called? + final String setterName = setterForGetter(getter.getName()); + if (setterName == null) continue; + + // get the setter method on the destination object + Method setter = null; + if (!isMap) { + try { + setter = dest.getClass().getMethod(setterName, getter.getReturnType()); + } catch (Exception e) { + log.debug("copy: setter not found: " + setterName); + continue; + } + } + + // do not copy null fields (should this be configurable?) + final Object srcValue = getter.invoke(src); + if (srcValue == null) continue; + + // does the dest have a getter? if so grab the current value + Object destValue = null; + try { + if (isMap) { + destValue = ((Map) dest).get(fieldName); + } else { + destValue = getter.invoke(dest); + } + } catch (Exception e) { + log.debug("copy: error calling getter on dest: "+e); + } + + // copy the value from src to dest, if it's different + if (!srcValue.equals(destValue)) { + if (isMap) { + ((Map) dest).put(fieldName, srcValue); + } else { + setter.invoke(dest, srcValue); + } + copyCount++; + } + } + } catch (Exception e) { + throw new IllegalArgumentException("Error copying "+dest.getClass().getSimpleName()+" from src="+src+": "+e, e); + } + return copyCount; + } + + private static boolean isIgnored(T o, String fieldName, Method getter) { + Field field = null; + try { + field = o.getClass().getDeclaredField(fieldName); + } catch (NoSuchFieldException ignored) {} + return isIgnored(getter.getAnnotations()) || (field != null && isIgnored(field.getAnnotations())); + } + + private static boolean isIgnored(Annotation[] annotations) { + if (annotations != null) { + for (Annotation a : annotations) { + final Class[] interfaces = a.getClass().getInterfaces(); + if (interfaces != null) { + for (Class i : interfaces) { + if (i.getSimpleName().equals("Transient")) { + return true; + } + } + } + } + } + return false; + } + + public static String fieldName(String method) { + if (method.startsWith("get")) return uncapitalize(method.substring(3)); + if (method.startsWith("set")) return uncapitalize(method.substring(3)); + if (method.startsWith("is")) return uncapitalize(method.substring(2)); + return null; + } + + public static String setterForGetter(String getter) { + if (getter.startsWith("get")) return "set"+getter.substring(3); + if (getter.startsWith("is")) return "set"+getter.substring(2); + return null; + } + + /** + * Call setters on an object based on keys and values in a Map + * @param dest destination object + * @param src map of field name -> value + * @param type of object + * @return the destination object + */ + public static T copyFromMap (T dest, Map src) { + return copyFromMap(dest, src, null); + } + + public static T copyFromMap (T dest, Map src, String[] exclude) { + for (Map.Entry entry : src.entrySet()) { + final String key = entry.getKey(); + if (exclude != null && ArrayUtils.contains(exclude, key)) continue; + final Object value = entry.getValue(); + if (value != null && Map.class.isAssignableFrom(value.getClass())) { + if (hasGetter(dest, key)) { + Map m = (Map) value; + if (m.isEmpty()) continue; + if (m.keySet().iterator().next().getClass().equals(String.class)) { + copyFromMap(get(dest, key), (Map) m); + } else { + log.info("copyFromMap: not recursively copying Map (has non-String keys): " + key); + } + } + } else { + if (Map.class.isAssignableFrom(dest.getClass())) {// || dest.getClass().getName().equals(HashMap.class.getName())) { + ((Map) dest).put(key, value); + } else { + if (hasSetter(dest, key, value.getClass())) { + set(dest, key, value); + } else { + final Class pc = getPrimitiveClass(value.getClass()); + if (pc != null && hasSetter(dest, key, pc)) { + set(dest, key, value); + } else { + log.info("copyFromMap: skipping uncopyable property: "+key); + } + } + } + } + } + return dest; + } + + public static Class getPrimitiveClass(Class clazz) { + if (clazz.isArray()) return arrayClass(getPrimitiveClass(clazz.getComponentType())); + switch (clazz.getSimpleName()) { + case "Long": return long.class; + case "Integer": return int.class; + case "Short": return short.class; + case "Double": return double.class; + case "Float": return float.class; + case "Boolean": return boolean.class; + case "Character": return char.class; + default: return null; + } + } + + public static final String[] TO_MAP_STANDARD_EXCLUDES = {"declaringClass", "class"}; + + /** + * Make a copy of the object, assuming its class has a copy constructor + * @param thing The thing to copy + * @return A copy of the object, created using the thing's copy constructor + */ + public static Map toMap(Object thing) { return toMap(thing, null, TO_MAP_STANDARD_EXCLUDES); } + + public static Map toMap(Object thing, String[] fields) { return toMap(thing, fields, TO_MAP_STANDARD_EXCLUDES); } + + public static Map toMap(Object thing, String[] fields, String[] exclude) { + final Map map = new HashMap<>(); + copy(map, thing, fields, exclude); + return map; + } + + /** + * Find the concrete class for the first declared parameterized class variable + * @param clazz The class to search for parameterized types + * @return The first concrete class for a parameterized type found in clazz + */ + public static Class getFirstTypeParam(Class clazz) { return getTypeParam(clazz, 0); } + + public static Class getTypeParam(Class clazz, int index) { + // todo: add a cache on this thing... could do wonders + Class check = clazz; + while (check.getGenericSuperclass() == null || !(check.getGenericSuperclass() instanceof ParameterizedType)) { + check = check.getSuperclass(); + if (check.equals(Object.class)) die("getTypeParam("+clazz.getName()+"): no type parameters found"); + } + final ParameterizedType parameterizedType = (ParameterizedType) check.getGenericSuperclass(); + final Type[] actualTypeArguments = parameterizedType.getActualTypeArguments(); + if (index >= actualTypeArguments.length) die("getTypeParam("+clazz.getName()+"): "+actualTypeArguments.length+" type parameters found, index "+index+" out of bounds"); + if (actualTypeArguments[index] instanceof Class) return (Class) actualTypeArguments[index]; + if (actualTypeArguments[index] instanceof ParameterizedType) return (Class) ((ParameterizedType) actualTypeArguments[index]).getRawType(); + return (Class) ((Type) actualTypeArguments[index]).getClass(); + } + + /** + * Find the concrete class for a parameterized class variable. + * @param clazz The class to start searching. Search will continue up through superclasses + * @param impl The type (or a supertype) of the parameterized class variable + * @return The first concrete class found that is assignable to an instance of impl + */ + public static Class getFirstTypeParam(Class clazz, Class impl) { + // todo: add a cache on this thing... could do wonders + Class check = clazz; + while (check != null && !check.equals(Object.class)) { + Class superCheck = check; + Type superType = superCheck.getGenericSuperclass(); + while (superType != null && !superType.equals(Object.class)) { + if (superType instanceof ParameterizedType) { + final ParameterizedType ptype = (ParameterizedType) superType; + final Class rawType = (Class) ptype.getRawType(); + if (impl.isAssignableFrom(rawType)) { + return (Class) rawType; + } + for (Type t : ptype.getActualTypeArguments()) { + if (impl.isAssignableFrom((Class) t)) { + return (Class) t; + } + } + + } else if (superType instanceof Class) { + superType = ((Class) superType).getGenericSuperclass(); + } + } + check = check.getSuperclass(); + } + return null; + } + + /** + * Call a getter. getXXX and isXXX will both be checked. + * @param object the object to call get(field) on + * @param field the field name + * @return the value of the field + * @throws IllegalArgumentException If no getter for the field exists + */ + public static Object get(Object object, String field) { + Object target = object; + for (String token : field.split("\\.")) { + if (target == null) return null; + target = invoke_get(target, token); + } + return target; + } + + public static T get(Object object, String field, T defaultValue) { + try { + final Object val = get(object, field); + return val == null ? defaultValue : (T) val; + } catch (Exception e) { + log.warn("get: "+e); + return defaultValue; + } + } + + public static boolean isGetter(Method method) { + return method.getName().startsWith("get") || method.getName().startsWith("is") && method.getParameters().length == 0; + } + + public static boolean hasGetter(Object object, String field) { + Object target = object; + try { + for (String token : field.split("\\.")) { + final String methodName = getAccessorMethodName(Accessor.get, token); + target = MethodUtils.invokeExactMethod(target, methodName, null); + } + } catch (NoSuchMethodException e) { + return false; + } catch (Exception e) { + return false; + } + return true; + } + + public static Class getterType(Object object, String field) { + try { + final Object o = get(object, field); + if (o == null) return die("getterType: cannot determine field type, value was null"); + return o.getClass(); + + } catch (Exception e) { + return die("getterType: simple get failed: "+e, e); + } + } + + /** + * Call a setter + * @param object the object to call set(field) on + * @param field the field name + * @param value the value to set + */ + public static void set(Object object, String field, Object value) { + set(object, field, value, value == null ? null : value.getClass()); + } + + /** + * Call a setter with a hint as to what the type should be + * @param object the object to call set(field) on + * @param field the field name + * @param value the value to set + * @param type type of the field + */ + public static void set(Object object, String field, Object value, Class type) { + if (type != null) { + if (value == null) { + setNull(object, field, type); + return; + } else if (!type.isAssignableFrom(value.getClass())) { + // if value is not assignable to type, then the type class should have a constructor for the value class + value = instantiate(type, value); + } + } + final String[] tokens = field.split("\\."); + Object target = getTarget(object, tokens); + if (target != null) invoke_set(target, tokens[tokens.length - 1], value); + } + + public static void setNull(Object object, String field, Class type) { + final String[] tokens = field.split("\\."); + Object target = getTarget(object, tokens); + if (target != null) invoke_set_null(target, tokens[tokens.length - 1], type); + } + + private static Object getTarget(Object object, String[] tokens) { + Object target = object; + for (int i=0; i setterCache = new ConcurrentHashMap<>(5000); + private static Map nullArgCache = new ConcurrentHashMap<>(5000); + + private static void invoke_set(Object target, String token, Object value) { + final String cacheKey = target.getClass().getName()+"."+token+"."+(value == null ? "null" : value.getClass().getName()); + final Method method = setterCache.computeIfAbsent(cacheKey, s -> { + final String methodName = getAccessorMethodName(Accessor.set, token); + Method found = null; + if (value == null) { + // try to find a single-arg method named methodName... + for (Method m : target.getClass().getMethods()) { + if (m.getName().equals(methodName) && m.getParameterTypes().length == 1) { + if (found != null) { + return die("invoke_set: value was null and multiple single-arg methods named " + methodName + " exist"); + } else { + found = m; + } + } + } + } else { + try { + found = MethodUtils.getMatchingAccessibleMethod(target.getClass(), methodName, new Class[]{value.getClass()}); + } catch (Exception e) { + return die("Error calling " + methodName + ": " + e); + } + } + return found != null ? found : die("invoke_set: no method " + methodName + " found on target: " + target); + }); + if (value == null) { + try { + final Object[] nullArg = nullArgCache.computeIfAbsent(method.getParameterTypes()[0], type -> new Object[] {getNullArgument(type)}); + method.invoke(target, nullArg); + } catch (Exception e) { + die("Error calling " + method.getName() + " on target: " + target + " - " + e); + } + } else { + try { + MethodUtils.invokeMethod(target, method.getName(), value); + } catch (Exception e) { + die("Error calling " + method.getName() + ": " + e); + } + } + } + + private static void invoke_set_null(Object target, String token, Class type) { + final String methodName = getAccessorMethodName(Accessor.set, token); + try { + MethodUtils.invokeMethod(target, methodName, new Object[] {getNullArgument(type)}, new Class[] { type }); + } catch (Exception e) { + die("Error calling "+methodName+": "+e); + } + } + + private static Object getNullArgument(Class clazz) { + if (clazz.isPrimitive()) { + switch (clazz.getName()) { + case "boolean": return false; + case "byte": return (byte) 0; + case "short": return (short) 0; + case "char": return (char) 0; + case "int": return (int) 0; + case "long": return (long) 0; + case "float": return (float) 0; + case "double": return (double) 0; + default: return die("instantiate: unrecognized primitive type: "+clazz.getName()); + } + } + return null; + } + + // methods below forked from dropwizard-- https://github.com/codahale/dropwizard + + /** + * Finds the type parameter for the given class. + * + * @param klass a parameterized class + * @return the class's type parameter + */ + public static Class getTypeParameter(Class klass) { + return getTypeParameter(klass, Object.class); + } + + /** + * Finds the type parameter for the given class which is assignable to the bound class. + * + * @param klass a parameterized class + * @param bound the type bound + * @param the type bound + * @return the class's type parameter + */ + @SuppressWarnings("unchecked") + public static Class getTypeParameter(Class klass, Class bound) { + Type t = checkNotNull(klass); + while (t instanceof Class) { + t = ((Class) t).getGenericSuperclass(); + } + /* This is not guaranteed to work for all cases with convoluted piping + * of type parameters: but it can at least resolve straight-forward + * extension with single type parameter (as per [Issue-89]). + * And when it fails to do that, will indicate with specific exception. + */ + if (t instanceof ParameterizedType) { + // should typically have one of type parameters (first one) that matches: + for (Type param : ((ParameterizedType) t).getActualTypeArguments()) { + if (param instanceof Class) { + final Class cls = determineClass(bound, param); + if (cls != null) { return cls; } + } + else if (param instanceof TypeVariable) { + for (Type paramBound : ((TypeVariable) param).getBounds()) { + if (paramBound instanceof Class) { + final Class cls = determineClass(bound, paramBound); + if (cls != null) { return cls; } + } + } + } + } + } + return die("Cannot figure out type parameterization for " + klass.getName()); + } + + @SuppressWarnings("unchecked") + private static Class determineClass(Class bound, Type candidate) { + if (candidate instanceof Class) { + final Class cls = (Class) candidate; + if (bound.isAssignableFrom(cls)) { + return (Class) cls; + } + } + + return null; + } + + public static void close(Object o) throws Exception { + if (o == null) return; + if (o instanceof Closeable) { + ((Closeable) o).close(); + + } else { + final Method closeMethod = o.getClass().getMethod("close", (Class[]) null); + if (closeMethod == null) die("no close method found on " + o.getClass().getName()); + closeMethod.invoke(o); + } + } + + public static void closeQuietly(Object o) { + if (o == null) return; + try { + close(o); + } catch (Exception e) { + log.warn("close: error closing: "+e); + } + } + + @NoArgsConstructor @AllArgsConstructor + public static class Setter { + @Getter protected String field; + @Getter protected String value; + public void set(T data) { ReflectionUtil.set(data, field, value); } + @Override public String toString() { return getClass().getName() + '{' + field + ", " + value + '}'; } + } + + // adapted from https://stackoverflow.com/a/2924426/1251543 + private static class CallerInspector extends SecurityManager { + public String getCallerClassName() { return getClassContext()[2].getName(); } + public String getCallerClassName(int depth) { return getClassContext()[depth].getName(); } + } + private final static CallerInspector callerInspector = new CallerInspector(); + + public static String callerClassName() { return callerInspector.getCallerClassName(); } + public static String callerClassName(int depth) { return callerInspector.getCallerClassName(depth); } + public static String callerClassName(String match) { + final StackTraceElement s = callerFrame(match); + return s == null ? "callerClassName: no match: "+match : s.getMethodName(); + } + + public static String callerMethodName() { return new Throwable().getStackTrace()[2].getMethodName(); } + public static String callerMethodName(int depth) { return new Throwable().getStackTrace()[depth].getMethodName(); } + public static String callerMethodName(String match) { + final StackTraceElement s = callerFrame(match); + return s == null ? "callerMethodName: no match: "+match : s.getMethodName(); + } + + public static String caller () { + final StackTraceElement[] t = new Throwable().getStackTrace(); + if (t == null || t.length == 0) return "caller: NO STACK TRACE!"; + return caller(t[Math.max(t.length-1, 2)]); + } + + public static String caller (int depth) { + final StackTraceElement[] t = new Throwable().getStackTrace(); + if (t == null || t.length == 0) return "caller: NO STACK TRACE!"; + return caller(t[Math.min(depth, t.length-1)]); + } + + public static String caller(String match) { + final StackTraceElement s = callerFrame(match); + return s == null ? "caller: no match: "+match : caller(s); + } + + public static StackTraceElement callerFrame(String match) { + final StackTraceElement[] t = new Throwable().getStackTrace(); + if (t == null || t.length == 0) return null; + for (StackTraceElement s : t) if (caller(s).contains(match)) return s; + return null; + } + + public static String caller(StackTraceElement s) { return s.getClassName() + "." + s.getMethodName() + ":" + s.getLineNumber(); } + + /** + * Replace any string values with their transformed values + * @param map a map of things + * @param transformer a transformer + * @return the same map, but if any value was a string, the transformer has been applied to it. + */ + public static Map transformStrings(Map map, Transformer transformer) { + if (empty(map)) return map; + final Map setOps = new HashMap(); + for (Object entry : map.entrySet()) { + final Map.Entry e = (Map.Entry) entry; + if (e.getValue() instanceof String) { + setOps.put(e.getKey(), transformer.transform(e.getValue()).toString()); + } else if (e.getValue() instanceof Map) { + setOps.put(e.getKey(), transformStrings((Map) e.getValue(), transformer)); + } + } + for (Object entry : setOps.entrySet()) { + final Map.Entry e = (Map.Entry) entry; + map.put(e.getKey(), e.getValue()); + } + return map; + } + +} diff --git a/src/main/java/org/cobbzilla/util/security/CryptStream.java b/src/main/java/org/cobbzilla/util/security/CryptStream.java new file mode 100644 index 0000000..fdaf5f2 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/security/CryptStream.java @@ -0,0 +1,81 @@ +package org.cobbzilla.util.security; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.bouncycastle.jce.provider.BouncyCastleProvider; + +import javax.crypto.Cipher; +import javax.crypto.CipherInputStream; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.Security; +import java.security.spec.AlgorithmParameterSpec; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.security.ShaUtil.sha256; + +@AllArgsConstructor +public class CryptStream { + + public static final int GCM_TAG_SIZE = 128; + public static final int BUFFER_SIZE = 8192; + + private final String password; + + @Getter(lazy=true) private final SecretKeySpec secretKey = new SecretKeySpec(sha256(password), "AES"); + + static { Security.addProvider(new BouncyCastleProvider()); } + + private static Cipher newCipher() { + try { + return Cipher.getInstance("AES/GCM/NoPadding", "BC"); + } catch (Exception e) { + return die("newCipher: "+e); + } + } + + private AlgorithmParameterSpec getIv(byte[] salt) { + if (salt.length != GCM_TAG_SIZE) return die("getIv: expected "+GCM_TAG_SIZE+" salt bytes for GCM tag"); + return new GCMParameterSpec(GCM_TAG_SIZE, salt); + } + + protected Cipher getEncryptionCipher(byte[] salt, String aad) { + try { + final Cipher cipher = newCipher(); + cipher.init(Cipher.ENCRYPT_MODE, getSecretKey(), getIv(salt)); + cipher.updateAAD(aad.getBytes()); + return cipher; + } catch (Exception e) { + return die("getEncryptionCipher: "+e); + } + } + + protected Cipher getDecryptionCipher(byte[] salt, String aad) { + try { + final Cipher cipher = newCipher(); + cipher.init(Cipher.DECRYPT_MODE, getSecretKey(), getIv(salt)); + cipher.updateAAD(aad.getBytes()); + return cipher; + } catch (Exception e) { + return die("getDecryptionCipher: "+e); + } + } + + public InputStream wrapRead(InputStream in, byte[] salt, String aad) throws IOException { + if (!(in instanceof BufferedInputStream)) { + in = new BufferedInputStream(in, BUFFER_SIZE); + } + return new CipherInputStream(in, getDecryptionCipher(salt, aad)); + } + + public InputStream wrapWrite(InputStream in, byte[] salt, String aad) throws IOException { + if (!(in instanceof BufferedInputStream)) { + in = new BufferedInputStream(in, BUFFER_SIZE); + } + return new CipherInputStream(in, getEncryptionCipher(salt, aad)); + } + +} diff --git a/src/main/java/org/cobbzilla/util/security/Crypto.java b/src/main/java/org/cobbzilla/util/security/Crypto.java new file mode 100644 index 0000000..4c989d7 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/security/Crypto.java @@ -0,0 +1,20 @@ +package org.cobbzilla.util.security; + +import java.io.InputStream; +import java.io.OutputStream; + +public interface Crypto { + + public String encrypt (String plaintext); + + public String decrypt (String ciphertext); + + public void encrypt(InputStream in, OutputStream out); + + public byte[] decryptBytes(InputStream in); + + public InputStream decryptStream(InputStream in); + + public String getSecretKey(); + +} diff --git a/src/main/java/org/cobbzilla/util/security/CryptoSimple.java b/src/main/java/org/cobbzilla/util/security/CryptoSimple.java new file mode 100644 index 0000000..4662695 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/security/CryptoSimple.java @@ -0,0 +1,64 @@ +package org.cobbzilla.util.security; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.compress.utils.IOUtils; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.io.OutputStream; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.security.CryptoUtil.string_decrypt; +import static org.cobbzilla.util.security.CryptoUtil.string_encrypt; + +@Slf4j @AllArgsConstructor +public class CryptoSimple implements Crypto { + + @Getter @Setter private String secretKey = null; + public boolean hasSecretKey() { return !empty(secretKey); } + + @Override public String encrypt (String plaintext) { + if (empty(secretKey)) die("encrypt: key was not initialized"); + if (empty(plaintext)) return null; + return string_encrypt(plaintext, secretKey); + } + + @Override public String decrypt (String ciphertext) { + if (empty(secretKey)) die("decrypt: key was not initialized"); + if (empty(ciphertext)) return null; + return string_decrypt(ciphertext, secretKey); + } + + // todo - support stream-oriented encryption + @Override public void encrypt(InputStream in, OutputStream out) { + if (empty(secretKey)) die("encrypt: key was not initialized"); + try { + final byte[] ciphertext = CryptoUtil.encrypt(in, secretKey); + IOUtils.copy(new ByteArrayInputStream(ciphertext), out); + + } catch (Exception e) { + die("encryption failed: "+e, e); + } + } + + @Override public byte[] decryptBytes(InputStream in) { + if (empty(secretKey)) die("encrypt: key was not initialized"); + try { + return CryptoUtil.decrypt(in, secretKey); + } catch (Exception e) { + return die("decryption failed: "+e, e); + } + } + + @Override public InputStream decryptStream(InputStream in) { + try { + return CryptoUtil.decryptStream(in, secretKey); + } catch (Exception e) { + return die("decryption failed: "+e, e); + } + } +} diff --git a/src/main/java/org/cobbzilla/util/security/CryptoUtil.java b/src/main/java/org/cobbzilla/util/security/CryptoUtil.java new file mode 100644 index 0000000..f439404 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/security/CryptoUtil.java @@ -0,0 +1,128 @@ +package org.cobbzilla.util.security; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.RandomStringUtils; +import org.cobbzilla.util.string.Base64; + +import javax.crypto.Cipher; +import javax.crypto.CipherInputStream; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.Key; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.retry; +import static org.cobbzilla.util.string.StringUtil.UTF8; + +@Slf4j +public class CryptoUtil { + + public static final String CONFIG_BLOCK_CIPHER = "AES/CBC/PKCS5Padding"; + public static final String CONFIG_KEY_CIPHER = "AES"; + + public static final String RSA_PREFIX = "-----BEGIN RSA PRIVATE KEY-----"; + public static final String RSA_SUFFIX = "-----END RSA PRIVATE KEY-----"; + + private static final MessageDigest MESSAGE_DIGEST; + static { + try { + MESSAGE_DIGEST = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw (RuntimeException) die("error creating SHA-256 MessageDigest: "+e); + } + } + + public static byte[] toBytes(InputStream data) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOUtils.copy(data, out); + return out.toByteArray(); + } + + public static String extractRsa (String data) { + int startPos = data.indexOf(RSA_PREFIX); + if (startPos == -1) return null; + int endPos = data.indexOf(RSA_SUFFIX); + if (endPos == -1) return null; + return data.substring(startPos, endPos + RSA_SUFFIX.length()); + } + + public static byte[] encrypt (InputStream data, String passphrase) throws Exception { + return encrypt(toBytes(data), passphrase); + } + + public static byte[] encrypt (byte[] data, String passphrase) throws Exception { + final Cipher cipher = Cipher.getInstance(CONFIG_BLOCK_CIPHER); + final Key keySpec = new SecretKeySpec(sha256(passphrase), CONFIG_KEY_CIPHER); + final IvParameterSpec initVector = new IvParameterSpec(new byte[cipher.getBlockSize()]); + cipher.init(Cipher.ENCRYPT_MODE, keySpec, initVector); + return cipher.doFinal(data); + } + + public static byte[] sha256(String passphrase) throws Exception { + return ShaUtil.sha256(passphrase); + } + + public static byte[] decrypt (InputStream data, String passphrase) throws Exception { + return decrypt(toBytes(data), passphrase); + } + + public static byte[] decrypt (byte[] data, String passphrase) throws Exception { + final Cipher cipher = Cipher.getInstance(CONFIG_BLOCK_CIPHER); + final Key keySpec = new SecretKeySpec(sha256(passphrase), CONFIG_KEY_CIPHER); + final IvParameterSpec initVector = new IvParameterSpec(new byte[cipher.getBlockSize()]); + cipher.init(Cipher.DECRYPT_MODE, keySpec, initVector); + return cipher.doFinal(data); + } + + public static InputStream decryptStream(InputStream in, String passphrase) throws Exception { + final Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); + final Key keySpec = new SecretKeySpec(sha256(passphrase), CONFIG_KEY_CIPHER); + final IvParameterSpec initVector = new IvParameterSpec(new byte[cipher.getBlockSize()]); + cipher.init(Cipher.DECRYPT_MODE, keySpec, initVector); + return new CipherInputStream(in, cipher); + } + + public static byte[] encryptOrDie(byte[] data, String passphrase) { + try { return encrypt(data, passphrase); } catch (Exception e) { + return die("Error encrypting: "+e, e); + } + } + + private static final String PADDING_SUFFIX = "__PADDING__"; + + public static String pad(String data) throws Exception { return data + PADDING_SUFFIX + RandomStringUtils.random(128); } + + public static String unpad(String data) { + if (data == null) return null; + int paddingPos = data.indexOf(PADDING_SUFFIX); + if (paddingPos == -1) return null; + return data.substring(0, paddingPos); + } + + public static String string_encrypt(String data, String key) { + try { return Base64.encodeBytes(encryptOrDie(pad(data).getBytes(UTF8), key)); } catch (Exception e) { + return die("Error encrypting: "+e, e); + } + } + + public static String string_decrypt(String data, String key) { + try { return unpad(new String(decrypt(Base64.decode(data), key))); } catch (Exception e) { + return die("Error decrypting: "+e, e); + } + } + + public static String generatePassword(int length, int minDistinct) { + return retry(() -> { + final String password = randomAlphanumeric(length); + if (password.chars().distinct().count() >= minDistinct) return password; + return die("minDistinct not met"); + }, 10); + } +} diff --git a/src/main/java/org/cobbzilla/util/security/HashType.java b/src/main/java/org/cobbzilla/util/security/HashType.java new file mode 100644 index 0000000..a4a4c7e --- /dev/null +++ b/src/main/java/org/cobbzilla/util/security/HashType.java @@ -0,0 +1,10 @@ +package org.cobbzilla.util.security; + +import com.fasterxml.jackson.annotation.JsonCreator; + +public enum HashType { + + sha256; + + @JsonCreator public static HashType create(String value) { return valueOf(value.toLowerCase()); } +} diff --git a/src/main/java/org/cobbzilla/util/security/MD5Util.java b/src/main/java/org/cobbzilla/util/security/MD5Util.java new file mode 100644 index 0000000..e3f5a0a --- /dev/null +++ b/src/main/java/org/cobbzilla/util/security/MD5Util.java @@ -0,0 +1,90 @@ +package org.cobbzilla.util.security; + +import org.cobbzilla.util.string.StringUtil; +import org.slf4j.Logger; + +import java.io.*; +import java.security.DigestInputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; + +public class MD5Util { + + private MD5Util () {} + + public static byte[] getMD5 ( byte[] bytes ) { + return getMD5(bytes, 0, bytes.length); + } + public static byte[] getMD5 ( byte[] bytes, int start, int len ) { + try { + MessageDigest md5 = MessageDigest.getInstance("MD5"); + md5.update( bytes, start, len ); + return md5.digest(); + } catch (NoSuchAlgorithmException e) { + return die("Error calculating MD5: " + e); + } + } + + public static String md5hex (Logger log, File file) throws IOException { + int BUFSIZ = 4096; + try (FileInputStream fin = new FileInputStream(file)) { + MessageDigest md5 = MessageDigest.getInstance("MD5"); + BufferedInputStream in = new BufferedInputStream(fin); + byte[] buf = new byte[BUFSIZ]; + int bytesRead = in.read(buf); + while (bytesRead != -1) { + md5.update(buf, 0, bytesRead); + bytesRead = in.read(buf); + } + return StringUtil.tohex(md5.digest()); + + } catch (NoSuchAlgorithmException e) { + return die("Error calculating MD5: " + e); + } + + } + + public static String md5hex ( String s ) { + byte[] bytes = getMD5(s.getBytes()); + return StringUtil.tohex(bytes); + } + + public static String md5hex (MessageDigest md) { + return StringUtil.tohex(md.digest()); + } + + public static String md5hex (byte[] data) { + return md5hex(data, 0, data.length); + } + public static String md5hex (byte[] data, int start, int len) { + byte[] bytes = getMD5(data, start, len); + return StringUtil.tohex(bytes); + } + + public static final String[] HEX_DIGITS = {"0", "1", "2", "3", + "4", "5", "6", "7", + "8", "9", "a", "b", + "c", "d", "e", "f"}; + + + public static MD5InputStream getMD5InputStream (InputStream in) { + try { + return new MD5InputStream(in); + } catch (NoSuchAlgorithmException e) { + return die("Bad algorithm: " + e); + } + } + + public static final class MD5InputStream extends DigestInputStream { + + public MD5InputStream(InputStream stream) throws NoSuchAlgorithmException { + super(stream, MessageDigest.getInstance("MD5")); + } + + public String md5hex () { + return MD5Util.md5hex(getMessageDigest()); + } + } +} diff --git a/src/main/java/org/cobbzilla/util/security/RsaKeyPair.java b/src/main/java/org/cobbzilla/util/security/RsaKeyPair.java new file mode 100644 index 0000000..c3a8b9e --- /dev/null +++ b/src/main/java/org/cobbzilla/util/security/RsaKeyPair.java @@ -0,0 +1,176 @@ +package org.cobbzilla.util.security; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.*; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.io.TempDir; +import org.cobbzilla.util.string.Base64; + +import java.io.File; + +import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; +import static org.cobbzilla.util.daemon.ZillaRuntime.*; +import static org.cobbzilla.util.io.FileUtil.*; +import static org.cobbzilla.util.string.StringUtil.safeShellArg; +import static org.cobbzilla.util.system.CommandShell.execScript; + +@NoArgsConstructor @Accessors(chain=true) @EqualsAndHashCode(of={"publicKey"}) @Slf4j +public class RsaKeyPair { + + public static final int DEFAULT_EXPIRATION_DAYS = 30; + public static final int MAX_RETRIES = 5; + + @Getter @Setter private String publicKey; + + @JsonIgnore @Getter(lazy=true) private final String sshPublicKey = initSshPublicKey(); + private String initSshPublicKey() { + try { + @Cleanup final TempDir temp = new TempDir(); + secureFile(temp, "key", getPrivateKey()); + final String sshKey = execScript("cd "+abs(temp)+" && ssh-keygen -f key -y"); + return sshKey.startsWith("ssh-rsa ") ? sshKey : die("error: "+sshKey); + } catch (Exception e) { + return die("initSshPublicKey: "+e.getMessage()); + } + } + + @JsonIgnore @Getter @Setter private String privateKey; + public boolean hasPrivateKey () { return !empty(privateKey); } + + @JsonIgnore @Getter(lazy=true) private final String sshPrivateKey = initSshPrivateKey(); + + private String initSshPrivateKey() { + try { + @Cleanup final TempDir temp = new TempDir(); + secureFile(temp, "key", getPrivateKey()); + final String sshKey = execScript("cd "+abs(temp)+" && openssl rsa -in key"); + return sshKey.startsWith("-----BEGIN RSA PRIVATE KEY-----\n") ? sshKey : die("error: "+sshKey); + } catch (Exception e) { + return die("initSshPrivateKey: "+e.getMessage()); + } + } + + public static RsaKeyPair newRsaKeyPair() { + @Cleanup("delete") final TempDir temp = newRsaKeyDir(); + return new RsaKeyPair() + .setPrivateKey(toStringOrDie(new File(temp, PRIVATE_KEY_FILE))) + .setPublicKey(toStringOrDie(new File(temp, PUBLIC_KEY_FILE))); + } + + public static TempDir newRsaKeyDir() { + return newRsaKeyDir(DEFAULT_EXPIRATION_DAYS, getDefaultSubject(), 0, MAX_RETRIES); + } + + public static TempDir newRsaKeyDir(int daysUntilExpiration) { + return newRsaKeyDir(daysUntilExpiration, getDefaultSubject(), 1, MAX_RETRIES); + } + + public static TempDir newRsaKeyDir(int daysUntilExpiration, String subject) { + return newRsaKeyDir(daysUntilExpiration, subject, 1, MAX_RETRIES); + } + + public static final String PRIVATE_KEY_FILE = "rsa.key"; + public static final String PUBLIC_KEY_FILE = "rsa.cert"; + private static final String PARAM_SUBJECT = "@@SUBJECT@@"; + private static final String PARAM_DAYS = "@@days@@"; + + private static final String CMD_NEW_KEY + = "openssl req -nodes -x509 -sha256 -newkey rsa:4096 -keyout "+PRIVATE_KEY_FILE+" -out "+PUBLIC_KEY_FILE+" " + + "-days "+PARAM_DAYS+ " -subj \""+PARAM_SUBJECT+"\""; + + public static String getDefaultSubject () { + return "/C=AQ/ST=Ross Ice Shelf/O=cobbzilla.org/CN=key."+randomAlphanumeric(10)+".cobbzilla.org"; + } + + private static TempDir newRsaKeyDir(int daysUntilExpiration, String subject, int attempt, int maxKeyTries) { + + if (attempt > maxKeyTries) return die("newRsaKeyDir: too many failures"); + + final TempDir temp = new TempDir(); + try { + final String keyCommand = CMD_NEW_KEY + .replace(PARAM_DAYS, "" + daysUntilExpiration) + .replace(PARAM_SUBJECT, "" + safeShellArg(subject)); + execScript("cd "+abs(temp)+" && "+ keyCommand); + final File keyFile = new File(temp, PRIVATE_KEY_FILE); + final File certFile = new File(temp, PUBLIC_KEY_FILE); + if (!keyFile.exists() || keyFile.length() == 0 || !certFile.exists() || certFile.length() == 0) { + if (!temp.delete()) log.warn("newRsaKeyPair: error deleting: "+abs(temp)); + return die("newRsaKeyPair: key not created"); + } + + // verify the key actually works + final String rand = randomAlphanumeric(200); + final RsaKeyPair key = new RsaKeyPair() + .setPrivateKey(toStringOrDie(keyFile)) + .setPublicKey(toStringOrDie(certFile)); + if (!key.decrypt(key.encrypt(rand, key), key).equals(rand)) { + log.warn("newRsaKeyDir: bad key, regenerating"); + return newRsaKeyDir(daysUntilExpiration, subject, attempt+1, maxKeyTries); + } + } catch (Exception e) { + log.warn("newRsaKeyDir: error creating/checking key, regenerating: "+e); + return newRsaKeyDir(daysUntilExpiration, subject, attempt+1, maxKeyTries); + } + return temp; + } + + public RsaMessage encrypt(String data, RsaKeyPair recipient) { + return retry(() -> { + @Cleanup("delete") final TempDir temp = new TempDir(); + + secureFile(temp, "data", data); + secureFile(temp, "recipient.crt", recipient.getPublicKey()); + secureFile(temp, "sender.key", getPrivateKey()); + secureFile(temp, "sender.crt", getPublicKey()); + try { + execScript("cd "+abs(temp)+" && " + + // generate random symmetric key + "openssl rand -out secret.key 32 && " + + + // encrypt data with symmetric key + "openssl aes-256-cbc -salt -pbkdf2 -in data -out data.enc -pass file:secret.key && " + + + // encrypt sym key with recipient's public key + "openssl rsautl -encrypt -oaep -pubin -certin -keyform PEM -inkey recipient.crt -in secret.key -out secret.key.enc && " + + + // sign with sender's private key + "openssl dgst -sha256 -sign sender.key -out data.sig data"); + + return new RsaMessage() + .setPublicKey(getPublicKey()) + .setSymKey(Base64.readB64(temp, "secret.key.enc")) + .setData(Base64.readB64(temp, "data.enc")) + .setSignature(Base64.readB64(temp, "data.sig")); + } catch (Exception e) { + return die("encrypt: "+e); + } + }, MAX_RETRIES); + } + + public String decrypt(RsaMessage message, RsaKeyPair sender) { + + @Cleanup("delete") final TempDir temp = new TempDir(); + try { + secureFile(temp, "sender.crt", sender.getPublicKey()); + secureFile(temp, "recipient.key", getPrivateKey()); + secureFileB64(temp, "data.enc", message.getData()); + secureFileB64(temp, "secret.key.enc", message.getSymKey()); + secureFileB64(temp, "data.sig", message.getSignature()); + + execScript("cd "+abs(temp)+" && " + + // decrypt symmetric key with recipient's private key + "openssl rsautl -decrypt -oaep -inkey recipient.key -in secret.key.enc -out secret.key && " + + + // decrypt data with symmetric key + "openssl aes-256-cbc -d -salt -pbkdf2 -in data.enc -out data -pass file:secret.key && " + + + // verify signature with sender's public key + "openssl dgst -sha256 -verify <(openssl x509 -in sender.crt -pubkey -noout) -signature data.sig data"); + return toStringOrDie(new File(temp, "data")); + } catch (Exception e) { + return die("decrypt: "+e); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/cobbzilla/util/security/RsaMessage.java b/src/main/java/org/cobbzilla/util/security/RsaMessage.java new file mode 100644 index 0000000..1237cd6 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/security/RsaMessage.java @@ -0,0 +1,20 @@ +package org.cobbzilla.util.security; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; + +import static org.cobbzilla.util.reflect.ReflectionUtil.copy; + +@NoArgsConstructor @Accessors(chain=true) +public class RsaMessage { + + public RsaMessage (RsaMessage other) { copy(this, other); } + + @Getter @Setter private String publicKey; + @Getter @Setter private String symKey; + @Getter @Setter private String data; + @Getter @Setter private String signature; + +} diff --git a/src/main/java/org/cobbzilla/util/security/ShaUtil.java b/src/main/java/org/cobbzilla/util/security/ShaUtil.java new file mode 100644 index 0000000..2f2a30d --- /dev/null +++ b/src/main/java/org/cobbzilla/util/security/ShaUtil.java @@ -0,0 +1,112 @@ +package org.cobbzilla.util.security; + +import lombok.Cleanup; +import org.apache.commons.exec.CommandLine; +import org.cobbzilla.util.string.Base64; +import org.cobbzilla.util.string.StringUtil; +import org.cobbzilla.util.system.CommandResult; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLEncoder; +import java.security.DigestException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.string.StringUtil.split; +import static org.cobbzilla.util.system.Bytes.MB; +import static org.cobbzilla.util.system.CommandShell.exec; + +public class ShaUtil { + + private static MessageDigest md() throws NoSuchAlgorithmException { return MessageDigest.getInstance("SHA-256"); } + + public static byte[] sha256 (String data) { + try { + return sha256(data.getBytes(StringUtil.UTF8)); + } catch (Exception e) { + return die("sha256: bad data: "+e, e); + } + } + + public static byte[] sha256 (byte[] data) { + if (data == null) throw new NullPointerException("sha256: null argument"); + try { + return md().digest(data); + } catch (Exception e) { + return die("sha256: bad data: "+e, e); + } + } + + public static String sha256_hex (String data) { return StringUtil.tohex(sha256(data)); } + + public static String sha256_base64 (byte[] data) throws Exception { return Base64.encodeBytes(sha256(data)); } + + public static String sha256_filename (String data) throws Exception { + return sha256_filename(data.getBytes(StringUtil.UTF8cs)); + } + + public static String sha256_filename (byte[] data) { + try { + return URLEncoder.encode(Base64.encodeBytes(sha256(data)), StringUtil.UTF8); + } catch (Exception e) { + return die("sha256_filename: bad byte[] data: "+e, e); + } + } + + public static final long SHA256_FILE_USE_SHELL_THRESHHOLD = 10*MB; + + public static String sha256_file (String file) { return sha256_file(new File(file)); } + + public static String sha256_file (File file) { + final CommandResult result; + try { + if (file.length() < SHA256_FILE_USE_SHELL_THRESHHOLD) return sha256_file_java(file); + result = exec(new CommandLine("sha256sum").addArgument(abs(file), false)); + if (result.isZeroExitStatus()) return split(result.getStdout(), " ").get(0); + + } catch (Exception e) { + // if we tried the shell command, it may have failed, try the pure java version + if (file.length() > SHA256_FILE_USE_SHELL_THRESHHOLD) return sha256_file_java(file); + return die("sha256sum_file: Error calculating sha256 on " + abs(file) + ": " + e); + } + return die("sha256sum_file: sha256sum "+abs(file)+" exited with status "+result.getExitStatus()+", stderr="+result.getStderr()+", exception="+result.getExceptionString()); + } + + public static String sha256_file_java(File file) { + try { + @Cleanup final InputStream input = new FileInputStream(file); + final MessageDigest md = getMessageDigest(input); + return StringUtil.tohex(md.digest()); + } catch (Exception e) { + return die("Error calculating sha256 on " + abs(file) + ": " + e); + } + } + + public static String sha256_url (String urlString) throws Exception { + + final URL url = new URL(urlString); + final URLConnection urlConnection = url.openConnection(); + @Cleanup final InputStream input = urlConnection.getInputStream(); + final MessageDigest md = getMessageDigest(input); + + return StringUtil.tohex(md.digest()); + } + + public static MessageDigest getMessageDigest(InputStream input) throws NoSuchAlgorithmException, IOException, DigestException { + final byte[] buf = new byte[4096]; + final MessageDigest md = md(); + while (true) { + int read = input.read(buf, 0, buf.length); + if (read == -1) break; + md.update(buf, 0, read); + } + return md; + } +} diff --git a/src/main/java/org/cobbzilla/util/security/bcrypt/BCrypt.java b/src/main/java/org/cobbzilla/util/security/bcrypt/BCrypt.java new file mode 100644 index 0000000..c718b61 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/security/bcrypt/BCrypt.java @@ -0,0 +1,752 @@ +// Copyright (c) 2006 Damien Miller +// +// Permission to use, copy, modify, and distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +package org.cobbzilla.util.security.bcrypt; + +import java.io.UnsupportedEncodingException; + +import java.security.SecureRandom; + +/** + * BCrypt implements OpenBSD-style Blowfish password hashing using + * the scheme described in "A Future-Adaptable Password Scheme" by + * Niels Provos and David Mazieres. + *

+ * This password hashing system tries to thwart off-line password + * cracking using a computationally-intensive hashing algorithm, + * based on Bruce Schneier's Blowfish cipher. The work factor of + * the algorithm is parameterised, so it can be increased as + * computers get faster. + *

+ * Usage is really simple. To hash a password for the first time, + * call the hashpw method with a random salt, like this: + *

+ * + * String pw_hash = BCrypt.hashpw(plain_password, BCrypt.gensalt());
+ * + *

+ * To check whether a plaintext password matches one that has been + * hashed previously, use the checkpw method: + *

+ * + * if (BCrypt.checkpw(candidate_password, stored_hash))
+ *     System.out.println("It matches");
+ * else
+ *     System.out.println("It does not match");
+ * + *

+ * The gensalt() method takes an optional parameter (log_rounds) + * that determines the computational complexity of the hashing: + *

+ * + * String strong_salt = BCrypt.gensalt(10)
+ * String stronger_salt = BCrypt.gensalt(12)
+ * + *

+ * The amount of work increases exponentially (2**log_rounds), so + * each increment is twice as much work. The default log_rounds is + * 10, and the valid range is 4 to 31. + * + * @author Damien Miller + * @version 0.2 + */ +public class BCrypt { + // BCrypt parameters + private static final int GENSALT_DEFAULT_LOG2_ROUNDS = 10; + private static final int BCRYPT_SALT_LEN = 16; + + // Blowfish parameters + private static final int BLOWFISH_NUM_ROUNDS = 16; + + // Initial contents of key schedule + private static final int P_orig[] = { + 0x243f6a88, 0x85a308d3, 0x13198a2e, 0x03707344, + 0xa4093822, 0x299f31d0, 0x082efa98, 0xec4e6c89, + 0x452821e6, 0x38d01377, 0xbe5466cf, 0x34e90c6c, + 0xc0ac29b7, 0xc97c50dd, 0x3f84d5b5, 0xb5470917, + 0x9216d5d9, 0x8979fb1b + }; + private static final int S_orig[] = { + 0xd1310ba6, 0x98dfb5ac, 0x2ffd72db, 0xd01adfb7, + 0xb8e1afed, 0x6a267e96, 0xba7c9045, 0xf12c7f99, + 0x24a19947, 0xb3916cf7, 0x0801f2e2, 0x858efc16, + 0x636920d8, 0x71574e69, 0xa458fea3, 0xf4933d7e, + 0x0d95748f, 0x728eb658, 0x718bcd58, 0x82154aee, + 0x7b54a41d, 0xc25a59b5, 0x9c30d539, 0x2af26013, + 0xc5d1b023, 0x286085f0, 0xca417918, 0xb8db38ef, + 0x8e79dcb0, 0x603a180e, 0x6c9e0e8b, 0xb01e8a3e, + 0xd71577c1, 0xbd314b27, 0x78af2fda, 0x55605c60, + 0xe65525f3, 0xaa55ab94, 0x57489862, 0x63e81440, + 0x55ca396a, 0x2aab10b6, 0xb4cc5c34, 0x1141e8ce, + 0xa15486af, 0x7c72e993, 0xb3ee1411, 0x636fbc2a, + 0x2ba9c55d, 0x741831f6, 0xce5c3e16, 0x9b87931e, + 0xafd6ba33, 0x6c24cf5c, 0x7a325381, 0x28958677, + 0x3b8f4898, 0x6b4bb9af, 0xc4bfe81b, 0x66282193, + 0x61d809cc, 0xfb21a991, 0x487cac60, 0x5dec8032, + 0xef845d5d, 0xe98575b1, 0xdc262302, 0xeb651b88, + 0x23893e81, 0xd396acc5, 0x0f6d6ff3, 0x83f44239, + 0x2e0b4482, 0xa4842004, 0x69c8f04a, 0x9e1f9b5e, + 0x21c66842, 0xf6e96c9a, 0x670c9c61, 0xabd388f0, + 0x6a51a0d2, 0xd8542f68, 0x960fa728, 0xab5133a3, + 0x6eef0b6c, 0x137a3be4, 0xba3bf050, 0x7efb2a98, + 0xa1f1651d, 0x39af0176, 0x66ca593e, 0x82430e88, + 0x8cee8619, 0x456f9fb4, 0x7d84a5c3, 0x3b8b5ebe, + 0xe06f75d8, 0x85c12073, 0x401a449f, 0x56c16aa6, + 0x4ed3aa62, 0x363f7706, 0x1bfedf72, 0x429b023d, + 0x37d0d724, 0xd00a1248, 0xdb0fead3, 0x49f1c09b, + 0x075372c9, 0x80991b7b, 0x25d479d8, 0xf6e8def7, + 0xe3fe501a, 0xb6794c3b, 0x976ce0bd, 0x04c006ba, + 0xc1a94fb6, 0x409f60c4, 0x5e5c9ec2, 0x196a2463, + 0x68fb6faf, 0x3e6c53b5, 0x1339b2eb, 0x3b52ec6f, + 0x6dfc511f, 0x9b30952c, 0xcc814544, 0xaf5ebd09, + 0xbee3d004, 0xde334afd, 0x660f2807, 0x192e4bb3, + 0xc0cba857, 0x45c8740f, 0xd20b5f39, 0xb9d3fbdb, + 0x5579c0bd, 0x1a60320a, 0xd6a100c6, 0x402c7279, + 0x679f25fe, 0xfb1fa3cc, 0x8ea5e9f8, 0xdb3222f8, + 0x3c7516df, 0xfd616b15, 0x2f501ec8, 0xad0552ab, + 0x323db5fa, 0xfd238760, 0x53317b48, 0x3e00df82, + 0x9e5c57bb, 0xca6f8ca0, 0x1a87562e, 0xdf1769db, + 0xd542a8f6, 0x287effc3, 0xac6732c6, 0x8c4f5573, + 0x695b27b0, 0xbbca58c8, 0xe1ffa35d, 0xb8f011a0, + 0x10fa3d98, 0xfd2183b8, 0x4afcb56c, 0x2dd1d35b, + 0x9a53e479, 0xb6f84565, 0xd28e49bc, 0x4bfb9790, + 0xe1ddf2da, 0xa4cb7e33, 0x62fb1341, 0xcee4c6e8, + 0xef20cada, 0x36774c01, 0xd07e9efe, 0x2bf11fb4, + 0x95dbda4d, 0xae909198, 0xeaad8e71, 0x6b93d5a0, + 0xd08ed1d0, 0xafc725e0, 0x8e3c5b2f, 0x8e7594b7, + 0x8ff6e2fb, 0xf2122b64, 0x8888b812, 0x900df01c, + 0x4fad5ea0, 0x688fc31c, 0xd1cff191, 0xb3a8c1ad, + 0x2f2f2218, 0xbe0e1777, 0xea752dfe, 0x8b021fa1, + 0xe5a0cc0f, 0xb56f74e8, 0x18acf3d6, 0xce89e299, + 0xb4a84fe0, 0xfd13e0b7, 0x7cc43b81, 0xd2ada8d9, + 0x165fa266, 0x80957705, 0x93cc7314, 0x211a1477, + 0xe6ad2065, 0x77b5fa86, 0xc75442f5, 0xfb9d35cf, + 0xebcdaf0c, 0x7b3e89a0, 0xd6411bd3, 0xae1e7e49, + 0x00250e2d, 0x2071b35e, 0x226800bb, 0x57b8e0af, + 0x2464369b, 0xf009b91e, 0x5563911d, 0x59dfa6aa, + 0x78c14389, 0xd95a537f, 0x207d5ba2, 0x02e5b9c5, + 0x83260376, 0x6295cfa9, 0x11c81968, 0x4e734a41, + 0xb3472dca, 0x7b14a94a, 0x1b510052, 0x9a532915, + 0xd60f573f, 0xbc9bc6e4, 0x2b60a476, 0x81e67400, + 0x08ba6fb5, 0x571be91f, 0xf296ec6b, 0x2a0dd915, + 0xb6636521, 0xe7b9f9b6, 0xff34052e, 0xc5855664, + 0x53b02d5d, 0xa99f8fa1, 0x08ba4799, 0x6e85076a, + 0x4b7a70e9, 0xb5b32944, 0xdb75092e, 0xc4192623, + 0xad6ea6b0, 0x49a7df7d, 0x9cee60b8, 0x8fedb266, + 0xecaa8c71, 0x699a17ff, 0x5664526c, 0xc2b19ee1, + 0x193602a5, 0x75094c29, 0xa0591340, 0xe4183a3e, + 0x3f54989a, 0x5b429d65, 0x6b8fe4d6, 0x99f73fd6, + 0xa1d29c07, 0xefe830f5, 0x4d2d38e6, 0xf0255dc1, + 0x4cdd2086, 0x8470eb26, 0x6382e9c6, 0x021ecc5e, + 0x09686b3f, 0x3ebaefc9, 0x3c971814, 0x6b6a70a1, + 0x687f3584, 0x52a0e286, 0xb79c5305, 0xaa500737, + 0x3e07841c, 0x7fdeae5c, 0x8e7d44ec, 0x5716f2b8, + 0xb03ada37, 0xf0500c0d, 0xf01c1f04, 0x0200b3ff, + 0xae0cf51a, 0x3cb574b2, 0x25837a58, 0xdc0921bd, + 0xd19113f9, 0x7ca92ff6, 0x94324773, 0x22f54701, + 0x3ae5e581, 0x37c2dadc, 0xc8b57634, 0x9af3dda7, + 0xa9446146, 0x0fd0030e, 0xecc8c73e, 0xa4751e41, + 0xe238cd99, 0x3bea0e2f, 0x3280bba1, 0x183eb331, + 0x4e548b38, 0x4f6db908, 0x6f420d03, 0xf60a04bf, + 0x2cb81290, 0x24977c79, 0x5679b072, 0xbcaf89af, + 0xde9a771f, 0xd9930810, 0xb38bae12, 0xdccf3f2e, + 0x5512721f, 0x2e6b7124, 0x501adde6, 0x9f84cd87, + 0x7a584718, 0x7408da17, 0xbc9f9abc, 0xe94b7d8c, + 0xec7aec3a, 0xdb851dfa, 0x63094366, 0xc464c3d2, + 0xef1c1847, 0x3215d908, 0xdd433b37, 0x24c2ba16, + 0x12a14d43, 0x2a65c451, 0x50940002, 0x133ae4dd, + 0x71dff89e, 0x10314e55, 0x81ac77d6, 0x5f11199b, + 0x043556f1, 0xd7a3c76b, 0x3c11183b, 0x5924a509, + 0xf28fe6ed, 0x97f1fbfa, 0x9ebabf2c, 0x1e153c6e, + 0x86e34570, 0xeae96fb1, 0x860e5e0a, 0x5a3e2ab3, + 0x771fe71c, 0x4e3d06fa, 0x2965dcb9, 0x99e71d0f, + 0x803e89d6, 0x5266c825, 0x2e4cc978, 0x9c10b36a, + 0xc6150eba, 0x94e2ea78, 0xa5fc3c53, 0x1e0a2df4, + 0xf2f74ea7, 0x361d2b3d, 0x1939260f, 0x19c27960, + 0x5223a708, 0xf71312b6, 0xebadfe6e, 0xeac31f66, + 0xe3bc4595, 0xa67bc883, 0xb17f37d1, 0x018cff28, + 0xc332ddef, 0xbe6c5aa5, 0x65582185, 0x68ab9802, + 0xeecea50f, 0xdb2f953b, 0x2aef7dad, 0x5b6e2f84, + 0x1521b628, 0x29076170, 0xecdd4775, 0x619f1510, + 0x13cca830, 0xeb61bd96, 0x0334fe1e, 0xaa0363cf, + 0xb5735c90, 0x4c70a239, 0xd59e9e0b, 0xcbaade14, + 0xeecc86bc, 0x60622ca7, 0x9cab5cab, 0xb2f3846e, + 0x648b1eaf, 0x19bdf0ca, 0xa02369b9, 0x655abb50, + 0x40685a32, 0x3c2ab4b3, 0x319ee9d5, 0xc021b8f7, + 0x9b540b19, 0x875fa099, 0x95f7997e, 0x623d7da8, + 0xf837889a, 0x97e32d77, 0x11ed935f, 0x16681281, + 0x0e358829, 0xc7e61fd6, 0x96dedfa1, 0x7858ba99, + 0x57f584a5, 0x1b227263, 0x9b83c3ff, 0x1ac24696, + 0xcdb30aeb, 0x532e3054, 0x8fd948e4, 0x6dbc3128, + 0x58ebf2ef, 0x34c6ffea, 0xfe28ed61, 0xee7c3c73, + 0x5d4a14d9, 0xe864b7e3, 0x42105d14, 0x203e13e0, + 0x45eee2b6, 0xa3aaabea, 0xdb6c4f15, 0xfacb4fd0, + 0xc742f442, 0xef6abbb5, 0x654f3b1d, 0x41cd2105, + 0xd81e799e, 0x86854dc7, 0xe44b476a, 0x3d816250, + 0xcf62a1f2, 0x5b8d2646, 0xfc8883a0, 0xc1c7b6a3, + 0x7f1524c3, 0x69cb7492, 0x47848a0b, 0x5692b285, + 0x095bbf00, 0xad19489d, 0x1462b174, 0x23820e00, + 0x58428d2a, 0x0c55f5ea, 0x1dadf43e, 0x233f7061, + 0x3372f092, 0x8d937e41, 0xd65fecf1, 0x6c223bdb, + 0x7cde3759, 0xcbee7460, 0x4085f2a7, 0xce77326e, + 0xa6078084, 0x19f8509e, 0xe8efd855, 0x61d99735, + 0xa969a7aa, 0xc50c06c2, 0x5a04abfc, 0x800bcadc, + 0x9e447a2e, 0xc3453484, 0xfdd56705, 0x0e1e9ec9, + 0xdb73dbd3, 0x105588cd, 0x675fda79, 0xe3674340, + 0xc5c43465, 0x713e38d8, 0x3d28f89e, 0xf16dff20, + 0x153e21e7, 0x8fb03d4a, 0xe6e39f2b, 0xdb83adf7, + 0xe93d5a68, 0x948140f7, 0xf64c261c, 0x94692934, + 0x411520f7, 0x7602d4f7, 0xbcf46b2e, 0xd4a20068, + 0xd4082471, 0x3320f46a, 0x43b7d4b7, 0x500061af, + 0x1e39f62e, 0x97244546, 0x14214f74, 0xbf8b8840, + 0x4d95fc1d, 0x96b591af, 0x70f4ddd3, 0x66a02f45, + 0xbfbc09ec, 0x03bd9785, 0x7fac6dd0, 0x31cb8504, + 0x96eb27b3, 0x55fd3941, 0xda2547e6, 0xabca0a9a, + 0x28507825, 0x530429f4, 0x0a2c86da, 0xe9b66dfb, + 0x68dc1462, 0xd7486900, 0x680ec0a4, 0x27a18dee, + 0x4f3ffea2, 0xe887ad8c, 0xb58ce006, 0x7af4d6b6, + 0xaace1e7c, 0xd3375fec, 0xce78a399, 0x406b2a42, + 0x20fe9e35, 0xd9f385b9, 0xee39d7ab, 0x3b124e8b, + 0x1dc9faf7, 0x4b6d1856, 0x26a36631, 0xeae397b2, + 0x3a6efa74, 0xdd5b4332, 0x6841e7f7, 0xca7820fb, + 0xfb0af54e, 0xd8feb397, 0x454056ac, 0xba489527, + 0x55533a3a, 0x20838d87, 0xfe6ba9b7, 0xd096954b, + 0x55a867bc, 0xa1159a58, 0xcca92963, 0x99e1db33, + 0xa62a4a56, 0x3f3125f9, 0x5ef47e1c, 0x9029317c, + 0xfdf8e802, 0x04272f70, 0x80bb155c, 0x05282ce3, + 0x95c11548, 0xe4c66d22, 0x48c1133f, 0xc70f86dc, + 0x07f9c9ee, 0x41041f0f, 0x404779a4, 0x5d886e17, + 0x325f51eb, 0xd59bc0d1, 0xf2bcc18f, 0x41113564, + 0x257b7834, 0x602a9c60, 0xdff8e8a3, 0x1f636c1b, + 0x0e12b4c2, 0x02e1329e, 0xaf664fd1, 0xcad18115, + 0x6b2395e0, 0x333e92e1, 0x3b240b62, 0xeebeb922, + 0x85b2a20e, 0xe6ba0d99, 0xde720c8c, 0x2da2f728, + 0xd0127845, 0x95b794fd, 0x647d0862, 0xe7ccf5f0, + 0x5449a36f, 0x877d48fa, 0xc39dfd27, 0xf33e8d1e, + 0x0a476341, 0x992eff74, 0x3a6f6eab, 0xf4f8fd37, + 0xa812dc60, 0xa1ebddf8, 0x991be14c, 0xdb6e6b0d, + 0xc67b5510, 0x6d672c37, 0x2765d43b, 0xdcd0e804, + 0xf1290dc7, 0xcc00ffa3, 0xb5390f92, 0x690fed0b, + 0x667b9ffb, 0xcedb7d9c, 0xa091cf0b, 0xd9155ea3, + 0xbb132f88, 0x515bad24, 0x7b9479bf, 0x763bd6eb, + 0x37392eb3, 0xcc115979, 0x8026e297, 0xf42e312d, + 0x6842ada7, 0xc66a2b3b, 0x12754ccc, 0x782ef11c, + 0x6a124237, 0xb79251e7, 0x06a1bbe6, 0x4bfb6350, + 0x1a6b1018, 0x11caedfa, 0x3d25bdd8, 0xe2e1c3c9, + 0x44421659, 0x0a121386, 0xd90cec6e, 0xd5abea2a, + 0x64af674e, 0xda86a85f, 0xbebfe988, 0x64e4c3fe, + 0x9dbc8057, 0xf0f7c086, 0x60787bf8, 0x6003604d, + 0xd1fd8346, 0xf6381fb0, 0x7745ae04, 0xd736fccc, + 0x83426b33, 0xf01eab71, 0xb0804187, 0x3c005e5f, + 0x77a057be, 0xbde8ae24, 0x55464299, 0xbf582e61, + 0x4e58f48f, 0xf2ddfda2, 0xf474ef38, 0x8789bdc2, + 0x5366f9c3, 0xc8b38e74, 0xb475f255, 0x46fcd9b9, + 0x7aeb2661, 0x8b1ddf84, 0x846a0e79, 0x915f95e2, + 0x466e598e, 0x20b45770, 0x8cd55591, 0xc902de4c, + 0xb90bace1, 0xbb8205d0, 0x11a86248, 0x7574a99e, + 0xb77f19b6, 0xe0a9dc09, 0x662d09a1, 0xc4324633, + 0xe85a1f02, 0x09f0be8c, 0x4a99a025, 0x1d6efe10, + 0x1ab93d1d, 0x0ba5a4df, 0xa186f20f, 0x2868f169, + 0xdcb7da83, 0x573906fe, 0xa1e2ce9b, 0x4fcd7f52, + 0x50115e01, 0xa70683fa, 0xa002b5c4, 0x0de6d027, + 0x9af88c27, 0x773f8641, 0xc3604c06, 0x61a806b5, + 0xf0177a28, 0xc0f586e0, 0x006058aa, 0x30dc7d62, + 0x11e69ed7, 0x2338ea63, 0x53c2dd94, 0xc2c21634, + 0xbbcbee56, 0x90bcb6de, 0xebfc7da1, 0xce591d76, + 0x6f05e409, 0x4b7c0188, 0x39720a3d, 0x7c927c24, + 0x86e3725f, 0x724d9db9, 0x1ac15bb4, 0xd39eb8fc, + 0xed545578, 0x08fca5b5, 0xd83d7cd3, 0x4dad0fc4, + 0x1e50ef5e, 0xb161e6f8, 0xa28514d9, 0x6c51133c, + 0x6fd5c7e7, 0x56e14ec4, 0x362abfce, 0xddc6c837, + 0xd79a3234, 0x92638212, 0x670efa8e, 0x406000e0, + 0x3a39ce37, 0xd3faf5cf, 0xabc27737, 0x5ac52d1b, + 0x5cb0679e, 0x4fa33742, 0xd3822740, 0x99bc9bbe, + 0xd5118e9d, 0xbf0f7315, 0xd62d1c7e, 0xc700c47b, + 0xb78c1b6b, 0x21a19045, 0xb26eb1be, 0x6a366eb4, + 0x5748ab2f, 0xbc946e79, 0xc6a376d2, 0x6549c2c8, + 0x530ff8ee, 0x468dde7d, 0xd5730a1d, 0x4cd04dc6, + 0x2939bbdb, 0xa9ba4650, 0xac9526e8, 0xbe5ee304, + 0xa1fad5f0, 0x6a2d519a, 0x63ef8ce2, 0x9a86ee22, + 0xc089c2b8, 0x43242ef6, 0xa51e03aa, 0x9cf2d0a4, + 0x83c061ba, 0x9be96a4d, 0x8fe51550, 0xba645bd6, + 0x2826a2f9, 0xa73a3ae1, 0x4ba99586, 0xef5562e9, + 0xc72fefd3, 0xf752f7da, 0x3f046f69, 0x77fa0a59, + 0x80e4a915, 0x87b08601, 0x9b09e6ad, 0x3b3ee593, + 0xe990fd5a, 0x9e34d797, 0x2cf0b7d9, 0x022b8b51, + 0x96d5ac3a, 0x017da67d, 0xd1cf3ed6, 0x7c7d2d28, + 0x1f9f25cf, 0xadf2b89b, 0x5ad6b472, 0x5a88f54c, + 0xe029ac71, 0xe019a5e6, 0x47b0acfd, 0xed93fa9b, + 0xe8d3c48d, 0x283b57cc, 0xf8d56629, 0x79132e28, + 0x785f0191, 0xed756055, 0xf7960e44, 0xe3d35e8c, + 0x15056dd4, 0x88f46dba, 0x03a16125, 0x0564f0bd, + 0xc3eb9e15, 0x3c9057a2, 0x97271aec, 0xa93a072a, + 0x1b3f6d9b, 0x1e6321f5, 0xf59c66fb, 0x26dcf319, + 0x7533d928, 0xb155fdf5, 0x03563482, 0x8aba3cbb, + 0x28517711, 0xc20ad9f8, 0xabcc5167, 0xccad925f, + 0x4de81751, 0x3830dc8e, 0x379d5862, 0x9320f991, + 0xea7a90c2, 0xfb3e7bce, 0x5121ce64, 0x774fbe32, + 0xa8b6e37e, 0xc3293d46, 0x48de5369, 0x6413e680, + 0xa2ae0810, 0xdd6db224, 0x69852dfd, 0x09072166, + 0xb39a460a, 0x6445c0dd, 0x586cdecf, 0x1c20c8ae, + 0x5bbef7dd, 0x1b588d40, 0xccd2017f, 0x6bb4e3bb, + 0xdda26a7e, 0x3a59ff45, 0x3e350a44, 0xbcb4cdd5, + 0x72eacea8, 0xfa6484bb, 0x8d6612ae, 0xbf3c6f47, + 0xd29be463, 0x542f5d9e, 0xaec2771b, 0xf64e6370, + 0x740e0d8d, 0xe75b1357, 0xf8721671, 0xaf537d5d, + 0x4040cb08, 0x4eb4e2cc, 0x34d2466a, 0x0115af84, + 0xe1b00428, 0x95983a1d, 0x06b89fb4, 0xce6ea048, + 0x6f3f3b82, 0x3520ab82, 0x011a1d4b, 0x277227f8, + 0x611560b1, 0xe7933fdc, 0xbb3a792b, 0x344525bd, + 0xa08839e1, 0x51ce794b, 0x2f32c9b7, 0xa01fbac9, + 0xe01cc87e, 0xbcc7d1f6, 0xcf0111c3, 0xa1e8aac7, + 0x1a908749, 0xd44fbd9a, 0xd0dadecb, 0xd50ada38, + 0x0339c32a, 0xc6913667, 0x8df9317c, 0xe0b12b4f, + 0xf79e59b7, 0x43f5bb3a, 0xf2d519ff, 0x27d9459c, + 0xbf97222c, 0x15e6fc2a, 0x0f91fc71, 0x9b941525, + 0xfae59361, 0xceb69ceb, 0xc2a86459, 0x12baa8d1, + 0xb6c1075e, 0xe3056a0c, 0x10d25065, 0xcb03a442, + 0xe0ec6e0e, 0x1698db3b, 0x4c98a0be, 0x3278e964, + 0x9f1f9532, 0xe0d392df, 0xd3a0342b, 0x8971f21e, + 0x1b0a7441, 0x4ba3348c, 0xc5be7120, 0xc37632d8, + 0xdf359f8d, 0x9b992f2e, 0xe60b6f47, 0x0fe3f11d, + 0xe54cda54, 0x1edad891, 0xce6279cf, 0xcd3e7e6f, + 0x1618b166, 0xfd2c1d05, 0x848fd2c5, 0xf6fb2299, + 0xf523f357, 0xa6327623, 0x93a83531, 0x56cccd02, + 0xacf08162, 0x5a75ebb5, 0x6e163697, 0x88d273cc, + 0xde966292, 0x81b949d0, 0x4c50901b, 0x71c65614, + 0xe6c6c7bd, 0x327a140a, 0x45e1d006, 0xc3f27b9a, + 0xc9aa53fd, 0x62a80f00, 0xbb25bfe2, 0x35bdd2f6, + 0x71126905, 0xb2040222, 0xb6cbcf7c, 0xcd769c2b, + 0x53113ec0, 0x1640e3d3, 0x38abbd60, 0x2547adf0, + 0xba38209c, 0xf746ce76, 0x77afa1c5, 0x20756060, + 0x85cbfe4e, 0x8ae88dd8, 0x7aaaf9b0, 0x4cf9aa7e, + 0x1948c25c, 0x02fb8a8c, 0x01c36ae4, 0xd6ebe1f9, + 0x90d4f869, 0xa65cdea0, 0x3f09252d, 0xc208e69f, + 0xb74e6132, 0xce77e25b, 0x578fdfe3, 0x3ac372e6 + }; + + // bcrypt IV: "OrpheanBeholderScryDoubt" + static private final int bf_crypt_ciphertext[] = { + 0x4f727068, 0x65616e42, 0x65686f6c, + 0x64657253, 0x63727944, 0x6f756274 + }; + + // Table for Base64 encoding + static private final char base64_code[] = { + '.', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', + 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', + 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', + 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', + '6', '7', '8', '9' + }; + + // Table for Base64 decoding + static private final byte index_64[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 0, 1, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, -1, -1, + -1, -1, -1, -1, -1, 2, 3, 4, 5, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + -1, -1, -1, -1, -1, -1, 28, 29, 30, + 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, + 51, 52, 53, -1, -1, -1, -1, -1 + }; + + // Expanded Blowfish key + private int P[]; + private int S[]; + + /** + * Encode a byte array using bcrypt's slightly-modified base64 + * encoding scheme. Note that this is *not* compatible with + * the standard MIME-base64 encoding. + * + * @param d the byte array to encode + * @param len the number of bytes to encode + * @return base64-encoded string + * @exception IllegalArgumentException if the length is invalid + */ + private static String encode_base64(byte d[], int len) + throws IllegalArgumentException { + int off = 0; + StringBuilder rs = new StringBuilder(); + int c1, c2; + + if (len <= 0 || len > d.length) + throw new IllegalArgumentException ("Invalid len"); + + while (off < len) { + c1 = d[off++] & 0xff; + rs.append(base64_code[(c1 >> 2) & 0x3f]); + c1 = (c1 & 0x03) << 4; + if (off >= len) { + rs.append(base64_code[c1 & 0x3f]); + break; + } + c2 = d[off++] & 0xff; + c1 |= (c2 >> 4) & 0x0f; + rs.append(base64_code[c1 & 0x3f]); + c1 = (c2 & 0x0f) << 2; + if (off >= len) { + rs.append(base64_code[c1 & 0x3f]); + break; + } + c2 = d[off++] & 0xff; + c1 |= (c2 >> 6) & 0x03; + rs.append(base64_code[c1 & 0x3f]); + rs.append(base64_code[c2 & 0x3f]); + } + return rs.toString(); + } + + /** + * Look up the 3 bits base64-encoded by the specified character, + * range-checking againt conversion table + * @param x the base64-encoded value + * @return the decoded value of x + */ + private static byte char64(char x) { + if ((int)x < 0 || (int)x > index_64.length) + return -1; + return index_64[(int)x]; + } + + /** + * Decode a string encoded using bcrypt's base64 scheme to a + * byte array. Note that this is *not* compatible with + * the standard MIME-base64 encoding. + * @param s the string to decode + * @param maxolen the maximum number of bytes to decode + * @return an array containing the decoded bytes + * @throws IllegalArgumentException if maxolen is invalid + */ + private static byte[] decode_base64(String s, int maxolen) + throws IllegalArgumentException { + StringBuilder rs = new StringBuilder(); + int off = 0, slen = s.length(), olen = 0; + byte ret[]; + byte c1, c2, c3, c4, o; + + if (maxolen <= 0) + throw new IllegalArgumentException ("Invalid maxolen"); + + while (off < slen - 1 && olen < maxolen) { + c1 = char64(s.charAt(off++)); + c2 = char64(s.charAt(off++)); + if (c1 == -1 || c2 == -1) + break; + o = (byte)(c1 << 2); + o |= (c2 & 0x30) >> 4; + rs.append((char)o); + if (++olen >= maxolen || off >= slen) + break; + c3 = char64(s.charAt(off++)); + if (c3 == -1) + break; + o = (byte)((c2 & 0x0f) << 4); + o |= (c3 & 0x3c) >> 2; + rs.append((char)o); + if (++olen >= maxolen || off >= slen) + break; + c4 = char64(s.charAt(off++)); + o = (byte)((c3 & 0x03) << 6); + o |= c4; + rs.append((char)o); + ++olen; + } + + ret = new byte[olen]; + for (off = 0; off < olen; off++) + ret[off] = (byte)rs.charAt(off); + return ret; + } + + /** + * Blowfish encipher a single 64-bit block encoded as + * two 32-bit halves + * @param lr an array containing the two 32-bit half blocks + * @param off the position in the array of the blocks + */ + private final void encipher(int lr[], int off) { + int i, n, l = lr[off], r = lr[off + 1]; + + l ^= P[0]; + for (i = 0; i <= BLOWFISH_NUM_ROUNDS - 2;) { + // Feistel substitution on left word + n = S[(l >> 24) & 0xff]; + n += S[0x100 | ((l >> 16) & 0xff)]; + n ^= S[0x200 | ((l >> 8) & 0xff)]; + n += S[0x300 | (l & 0xff)]; + r ^= n ^ P[++i]; + + // Feistel substitution on right word + n = S[(r >> 24) & 0xff]; + n += S[0x100 | ((r >> 16) & 0xff)]; + n ^= S[0x200 | ((r >> 8) & 0xff)]; + n += S[0x300 | (r & 0xff)]; + l ^= n ^ P[++i]; + } + lr[off] = r ^ P[BLOWFISH_NUM_ROUNDS + 1]; + lr[off + 1] = l; + } + + /** + * Cycically extract a word of key material + * @param data the string to extract the data from + * @param offp a "pointer" (as a one-entry array) to the + * current offset into data + * @return the next word of material from data + */ + private static int streamtoword(byte data[], int offp[]) { + int i; + int word = 0; + int off = offp[0]; + + for (i = 0; i < 4; i++) { + word = (word << 8) | (data[off] & 0xff); + off = (off + 1) % data.length; + } + + offp[0] = off; + return word; + } + + /** + * Initialise the Blowfish key schedule + */ + private void init_key() { + P = (int[])P_orig.clone(); + S = (int[])S_orig.clone(); + } + + /** + * Key the Blowfish cipher + * @param key an array containing the key + */ + private void key(byte key[]) { + int i; + int koffp[] = { 0 }; + int lr[] = { 0, 0 }; + int plen = P.length, slen = S.length; + + for (i = 0; i < plen; i++) + P[i] = P[i] ^ streamtoword(key, koffp); + + for (i = 0; i < plen; i += 2) { + encipher(lr, 0); + P[i] = lr[0]; + P[i + 1] = lr[1]; + } + + for (i = 0; i < slen; i += 2) { + encipher(lr, 0); + S[i] = lr[0]; + S[i + 1] = lr[1]; + } + } + + /** + * Perform the "enhanced key schedule" step described by + * Provos and Mazieres in "A Future-Adaptable Password Scheme" + * http://www.openbsd.org/papers/bcrypt-paper.ps + * @param data salt information + * @param key password information + */ + private void ekskey(byte data[], byte key[]) { + int i; + int koffp[] = { 0 }, doffp[] = { 0 }; + int lr[] = { 0, 0 }; + int plen = P.length, slen = S.length; + + for (i = 0; i < plen; i++) + P[i] = P[i] ^ streamtoword(key, koffp); + + for (i = 0; i < plen; i += 2) { + lr[0] ^= streamtoword(data, doffp); + lr[1] ^= streamtoword(data, doffp); + encipher(lr, 0); + P[i] = lr[0]; + P[i + 1] = lr[1]; + } + + for (i = 0; i < slen; i += 2) { + lr[0] ^= streamtoword(data, doffp); + lr[1] ^= streamtoword(data, doffp); + encipher(lr, 0); + S[i] = lr[0]; + S[i + 1] = lr[1]; + } + } + + /** + * Perform the central password hashing step in the + * bcrypt scheme + * @param password the password to hash + * @param salt the binary salt to hash with the password + * @param log_rounds the binary logarithm of the number + * of rounds of hashing to apply + * @return an array containing the binary hashed password + */ + private byte[] crypt_raw(byte password[], byte salt[], int log_rounds) { + int rounds, i, j; + int cdata[] = (int[])bf_crypt_ciphertext.clone(); + int clen = cdata.length; + byte ret[]; + + if (log_rounds < 4 || log_rounds > 31) + throw new IllegalArgumentException ("Bad number of rounds"); + rounds = 1 << log_rounds; + if (salt.length != BCRYPT_SALT_LEN) + throw new IllegalArgumentException ("Bad salt length"); + + init_key(); + ekskey(salt, password); + for (i = 0; i < rounds; i++) { + key(password); + key(salt); + } + + for (i = 0; i < 64; i++) { + for (j = 0; j < (clen >> 1); j++) + encipher(cdata, j << 1); + } + + ret = new byte[clen * 4]; + for (i = 0, j = 0; i < clen; i++) { + ret[j++] = (byte)((cdata[i] >> 24) & 0xff); + ret[j++] = (byte)((cdata[i] >> 16) & 0xff); + ret[j++] = (byte)((cdata[i] >> 8) & 0xff); + ret[j++] = (byte)(cdata[i] & 0xff); + } + return ret; + } + + /** + * Hash a password using the OpenBSD bcrypt scheme + * @param password the password to hash + * @param salt the salt to hash with (perhaps generated + * using BCrypt.gensalt) + * @return the hashed password + */ + public static String hashpw(String password, String salt) { + BCrypt B; + String real_salt; + byte passwordb[], saltb[], hashed[]; + char minor = (char)0; + int rounds, off = 0; + StringBuilder rs = new StringBuilder(); + + if (salt.charAt(0) != '$' || salt.charAt(1) != '2') + throw new IllegalArgumentException ("Invalid salt version"); + if (salt.charAt(2) == '$') + off = 3; + else { + minor = salt.charAt(2); + if (minor != 'a' || salt.charAt(3) != '$') + throw new IllegalArgumentException ("Invalid salt revision"); + off = 4; + } + + // Extract number of rounds + if (salt.charAt(off + 2) > '$') + throw new IllegalArgumentException ("Missing salt rounds"); + rounds = Integer.parseInt(salt.substring(off, off + 2)); + + real_salt = salt.substring(off + 3, off + 25); + try { + passwordb = (password + (minor >= 'a' ? "\000" : "")).getBytes("UTF-8"); + } catch (UnsupportedEncodingException uee) { + throw new AssertionError("UTF-8 is not supported"); + } + + saltb = decode_base64(real_salt, BCRYPT_SALT_LEN); + + B = new BCrypt(); + hashed = B.crypt_raw(passwordb, saltb, rounds); + + rs.append("$2"); + if (minor >= 'a') + rs.append(minor); + rs.append("$"); + if (rounds < 10) + rs.append("0"); + rs.append(Integer.toString(rounds)); + rs.append("$"); + rs.append(encode_base64(saltb, saltb.length)); + rs.append(encode_base64(hashed, + bf_crypt_ciphertext.length * 4 - 1)); + return rs.toString(); + } + + /** + * Generate a salt for use with the BCrypt.hashpw() method + * @param log_rounds the log2 of the number of rounds of + * hashing to apply - the work factor therefore increases as + * 2**log_rounds. + * @param random an instance of SecureRandom to use + * @return an encoded salt value + */ + public static String gensalt(int log_rounds, SecureRandom random) { + StringBuilder rs = new StringBuilder(); + byte rnd[] = new byte[BCRYPT_SALT_LEN]; + + random.nextBytes(rnd); + + rs.append("$2a$"); + if (log_rounds < 10) + rs.append("0"); + rs.append(Integer.toString(log_rounds)); + rs.append("$"); + rs.append(encode_base64(rnd, rnd.length)); + return rs.toString(); + } + + /** + * Generate a salt for use with the BCrypt.hashpw() method + * @param log_rounds the log2 of the number of rounds of + * hashing to apply - the work factor therefore increases as + * 2**log_rounds. + * @return an encoded salt value + */ + public static String gensalt(int log_rounds) { + return gensalt(log_rounds, new SecureRandom()); + } + + /** + * Generate a salt for use with the BCrypt.hashpw() method, + * selecting a reasonable default for the number of hashing + * rounds to apply + * @return an encoded salt value + */ + public static String gensalt() { + return gensalt(GENSALT_DEFAULT_LOG2_ROUNDS); + } + + /** + * Check that a plaintext password matches a previously hashed + * one + * @param plaintext the plaintext password to verify + * @param hashed the previously-hashed password + * @return true if the passwords match, false otherwise + */ + public static boolean checkpw(String plaintext, String hashed) { + return (hashed.compareTo(hashpw(plaintext, hashed)) == 0); + } +} diff --git a/src/main/java/org/cobbzilla/util/security/bcrypt/BCryptUtil.java b/src/main/java/org/cobbzilla/util/security/bcrypt/BCryptUtil.java new file mode 100644 index 0000000..3993373 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/security/bcrypt/BCryptUtil.java @@ -0,0 +1,41 @@ +package org.cobbzilla.util.security.bcrypt; + +import lombok.extern.slf4j.Slf4j; + +import java.security.SecureRandom; + +@Slf4j +public class BCryptUtil { + + private static final SecureRandom random = new SecureRandom(); + + private static volatile Integer bcryptRounds = null; + + public synchronized static void setBcryptRounds(int rounds) { + if (bcryptRounds != null) { + log.warn("Cannot change bcryptRounds after initialization"); + return; + } + bcryptRounds = rounds < 4 ? 4 : rounds; // 4 is minimum bcrypt rounds + log.info("setBcryptRounds: initialized with "+bcryptRounds+" rounds (param was "+rounds+")"); + } + + public static Integer getBcryptRounds() { return bcryptRounds; } + + public static String hash(String password) { + return BCrypt.hashpw(password, BCrypt.gensalt(getBcryptRounds(), random)); + } + + public static void main (String[] args) { + int input = 0; + int rounds = 16; + try { + rounds = Integer.valueOf(args[0]); + input = 1; + } catch (Exception ignored) { + // whatever + } + setBcryptRounds(rounds); + System.out.println(hash(args[input])); + } +} diff --git a/src/main/java/org/cobbzilla/util/string/Base64.java b/src/main/java/org/cobbzilla/util/string/Base64.java new file mode 100644 index 0000000..7873bf6 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/string/Base64.java @@ -0,0 +1,2080 @@ +package org.cobbzilla.util.string; + +import java.io.File; +import java.io.IOException; + +/** + *

Encodes and decodes to and from Base64 notation.

+ *

Homepage: http://iharder.net/base64.

+ * + *

Example:

+ * + * String encoded = Base64.encode( myByteArray ); + *
+ * byte[] myByteArray = Base64.decode( encoded ); + * + *

The options parameter, which appears in a few places, is used to pass + * several pieces of information to the encoder. In the "higher level" methods such as + * encodeBytes( bytes, options ) the options parameter can be used to indicate such + * things as first gzipping the bytes before encoding them, not inserting linefeeds, + * and encoding using the URL-safe and Ordered dialects.

+ * + *

Note, according to RFC3548, + * Section 2.1, implementations should not add line feeds unless explicitly told + * to do so. I've got Base64 set to this behavior now, although earlier versions + * broke lines by default.

+ * + *

The constants defined in Base64 can be OR-ed together to combine options, so you + * might make a call like this:

+ * + * String encoded = Base64.encodeBytes( mybytes, Base64.GZIP | Base64.DO_BREAK_LINES ); + *

to compress the data before encoding it and then making the output have newline characters.

+ *

Also...

+ * String encoded = Base64.encodeBytes( crazyString.getBytes() ); + * + * + * + *

+ * Change Log: + *

+ *
    + *
  • v2.3.7 - Fixed subtle bug when base 64 input stream contained the + * value 01111111, which is an invalid base 64 character but should not + * throw an ArrayIndexOutOfBoundsException either. Led to discovery of + * mishandling (or potential for better handling) of other bad input + * characters. You should now get an IOException if you try decoding + * something that has bad characters in it.
    • + *
  • v2.3.6 - Fixed bug when breaking lines and the final byte of the encoded + * string ended in the last column; the buffer was not properly shrunk and + * contained an extra (null) byte that made it into the string.
    • + *
  • v2.3.5 - Fixed bug in {@link #encodeFromFile} where estimated buffer size + * was wrong for files of size 31, 34, and 37 bytes.
    • + *
  • v2.3.4 - Fixed bug when working with gzipped streams whereby flushing + * the Base64.OutputStream closed the Base64 encoding (by padding with equals + * signs) too soon. Also added an option to suppress the automatic decoding + * of gzipped streams. Also added experimental support for specifying a + * class loader when using the + * {@link #decodeToObject(java.lang.String, int, java.lang.ClassLoader)} + * method.
    • + *
  • v2.3.3 - Changed default char encoding to US-ASCII which reduces the internal Java + * footprint with its CharEncoders and so forth. Fixed some javadocs that were + * inconsistent. Removed imports and specified things like java.io.IOException + * explicitly inline.
    • + *
  • v2.3.2 - Reduced memory footprint! Finally refined the "guessing" of how big the + * final encoded data will be so that the code doesn't have to create two output + * arrays: an oversized initial one and then a final, exact-sized one. Big win + * when using the {@link #encodeBytesToBytes(byte[])} family of methods (and not + * using the gzip options which uses a different mechanism with streams and stuff).
    • + *
  • v2.3.1 - Added {@link #encodeBytesToBytes(byte[], int, int, int)} and some + * similar helper methods to be more efficient with memory by not returning a + * String but just a byte array.
    • + *
  • v2.3 - This is not a drop-in replacement! This is two years of comments + * and bug fixes queued up and finally executed. Thanks to everyone who sent + * me stuff, and I'm sorry I wasn't able to distribute your fixes to everyone else. + * Much bad coding was cleaned up including throwing exceptions where necessary + * instead of returning null values or something similar. Here are some changes + * that may affect you: + *
      + *
    • Does not break lines, by default. This is to keep in compliance with + * RFC3548.
      • + *
    • Throws exceptions instead of returning null values. Because some operations + * (especially those that may permit the GZIP option) use IO streams, there + * is a possiblity of an java.io.IOException being thrown. After some discussion and + * thought, I've changed the behavior of the methods to throw java.io.IOExceptions + * rather than return null if ever there's an error. I think this is more + * appropriate, though it will require some changes to your code. Sorry, + * it should have been done this way to begin with.
      • + *
    • Removed all references to System.out, System.err, and the like. + * Shame on me. All I can say is sorry they were ever there.
      • + *
    • Throws NullPointerExceptions and IllegalArgumentExceptions as needed + * such as when passed arrays are null or offsets are invalid.
      • + *
    • Cleaned up as much javadoc as I could to avoid any javadoc warnings. + * This was especially annoying before for people who were thorough in their + * own projects and then had gobs of javadoc warnings on this file.
      • + *
    + *
  • v2.2.1 - Fixed bug using URL_SAFE and ORDERED encodings. Fixed bug + * when using very small files (~< 40 bytes).
    • + *
  • v2.2 - Added some helper methods for encoding/decoding directly from + * one file to the next. Also added a main() method to support command line + * encoding/decoding from one file to the next. Also added these Base64 dialects: + *
      + *
    1. The default is RFC3548 format.
      2. + *
    2. Calling Base64.setFormat(Base64.BASE64_FORMAT.URLSAFE_FORMAT) generates + * URL and file name friendly format as described in Section 4 of RFC3548. + * http://www.faqs.org/rfcs/rfc3548.html
      3. + *
    3. Calling Base64.setFormat(Base64.BASE64_FORMAT.ORDERED_FORMAT) generates + * URL and file name friendly format that preserves lexical ordering as described + * in http://www.faqs.org/qa/rfcc-1940.html
      4. + *
    + * Special thanks to Jim Kellerman at http://www.powerset.com/ + * for contributing the new Base64 dialects. + *
    • + * + *
  • v2.1 - Cleaned up javadoc comments and unused variables and methods. Added + * some convenience methods for reading and writing to and from files.
    • + *
  • v2.0.2 - Now specifies UTF-8 encoding in places where the code fails on systems + * with other encodings (like EBCDIC).
    • + *
  • v2.0.1 - Fixed an error when decoding a single byte, that is, when the + * encoded data was a single byte.
    • + *
  • v2.0 - I got rid of methods that used booleans to set options. + * Now everything is more consolidated and cleaner. The code now detects + * when data that's being decoded is gzip-compressed and will decompress it + * automatically. Generally things are cleaner. You'll probably have to + * change some method calls that you were making to support the new + * options format (ints that you "OR" together).
    • + *
  • v1.5.1 - Fixed bug when decompressing and decoding to a + * byte[] using decode( String s, boolean gzipCompressed ). + * Added the ability to "suspend" encoding in the Output Stream so + * you can turn on and off the encoding if you need to embed base64 + * data in an otherwise "normal" stream (like an XML file).
    • + *
  • v1.5 - Output stream pases on flush() command but doesn't do anything itself. + * This helps when using GZIP streams. + * Added the ability to GZip-compress objects before encoding them.
    • + *
  • v1.4 - Added helper methods to read/write files.
    • + *
  • v1.3.6 - Fixed OutputStream.flush() so that 'position' is reset.
    • + *
  • v1.3.5 - Added flag to turn on and off line breaks. Fixed bug in input stream + * where last buffer being read, if not completely full, was not returned.
    • + *
  • v1.3.4 - Fixed when "improperly padded stream" error was thrown at the wrong time.
    • + *
  • v1.3.3 - Fixed I/O streams which were totally messed up.
    • + *
+ * + *

+ * I am placing this code in the Public Domain. Do with it as you will. + * This software comes with no guarantees or warranties but with + * plenty of well-wishing instead! + * Please visit http://iharder.net/base64 + * periodically to check for updates or to contribute improvements. + *

+ * + * @author Robert Harder + * @author rob@iharder.net + * @version 2.3.7 + */ +public class Base64 +{ + +/* ******** P U B L I C F I E L D S ******** */ + + + /** No options specified. Value is zero. */ + public final static int NO_OPTIONS = 0; + + /** Specify encoding in first bit. Value is one. */ + public final static int ENCODE = 1; + + + /** Specify decoding in first bit. Value is zero. */ + public final static int DECODE = 0; + + + /** Specify that data should be gzip-compressed in second bit. Value is two. */ + public final static int GZIP = 2; + + /** Specify that gzipped data should not be automatically gunzipped. */ + public final static int DONT_GUNZIP = 4; + + + /** Do break lines when encoding. Value is 8. */ + public final static int DO_BREAK_LINES = 8; + + /** + * Encode using Base64-like encoding that is URL- and Filename-safe as described + * in Section 4 of RFC3548: + * http://www.faqs.org/rfcs/rfc3548.html. + * It is important to note that data encoded this way is not officially valid Base64, + * or at the very least should not be called Base64 without also specifying that is + * was encoded using the URL- and Filename-safe dialect. + */ + public final static int URL_SAFE = 16; + + + /** + * Encode using the special "ordered" dialect of Base64 described here: + * http://www.faqs.org/qa/rfcc-1940.html. + */ + public final static int ORDERED = 32; + + +/* ******** P R I V A T E F I E L D S ******** */ + + + /** Maximum line length (76) of Base64 output. */ + private final static int MAX_LINE_LENGTH = 76; + + + /** The equals sign (=) as a byte. */ + private final static byte EQUALS_SIGN = (byte)'='; + + + /** The new line character (\n) as a byte. */ + private final static byte NEW_LINE = (byte)'\n'; + + + /** Preferred encoding. */ + private final static String PREFERRED_ENCODING = "US-ASCII"; + + + private final static byte WHITE_SPACE_ENC = -5; // Indicates white space in encoding + private final static byte EQUALS_SIGN_ENC = -1; // Indicates equals sign in encoding + + +/* ******** S T A N D A R D B A S E 6 4 A L P H A B E T ******** */ + + /** The 64 valid Base64 values. */ + /* Host platform me be something funny like EBCDIC, so we hardcode these values. */ + private final static byte[] _STANDARD_ALPHABET = { + (byte)'A', (byte)'B', (byte)'C', (byte)'D', (byte)'E', (byte)'F', (byte)'G', + (byte)'H', (byte)'I', (byte)'J', (byte)'K', (byte)'L', (byte)'M', (byte)'N', + (byte)'O', (byte)'P', (byte)'Q', (byte)'R', (byte)'S', (byte)'T', (byte)'U', + (byte)'V', (byte)'W', (byte)'X', (byte)'Y', (byte)'Z', + (byte)'a', (byte)'b', (byte)'c', (byte)'d', (byte)'e', (byte)'f', (byte)'g', + (byte)'h', (byte)'i', (byte)'j', (byte)'k', (byte)'l', (byte)'m', (byte)'n', + (byte)'o', (byte)'p', (byte)'q', (byte)'r', (byte)'s', (byte)'t', (byte)'u', + (byte)'v', (byte)'w', (byte)'x', (byte)'y', (byte)'z', + (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', + (byte)'6', (byte)'7', (byte)'8', (byte)'9', (byte)'+', (byte)'/' + }; + + + /** + * Translates a Base64 value to either its 6-bit reconstruction value + * or a negative number indicating some other meaning. + **/ + private final static byte[] _STANDARD_DECODABET = { + -9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 0 - 8 + -5,-5, // Whitespace: Tab and Linefeed + -9,-9, // Decimal 11 - 12 + -5, // Whitespace: Carriage Return + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 14 - 26 + -9,-9,-9,-9,-9, // Decimal 27 - 31 + -5, // Whitespace: Space + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 33 - 42 + 62, // Plus sign at decimal 43 + -9,-9,-9, // Decimal 44 - 46 + 63, // Slash at decimal 47 + 52,53,54,55,56,57,58,59,60,61, // Numbers zero through nine + -9,-9,-9, // Decimal 58 - 60 + -1, // Equals sign at decimal 61 + -9,-9,-9, // Decimal 62 - 64 + 0,1,2,3,4,5,6,7,8,9,10,11,12,13, // Letters 'A' through 'N' + 14,15,16,17,18,19,20,21,22,23,24,25, // Letters 'O' through 'Z' + -9,-9,-9,-9,-9,-9, // Decimal 91 - 96 + 26,27,28,29,30,31,32,33,34,35,36,37,38, // Letters 'a' through 'm' + 39,40,41,42,43,44,45,46,47,48,49,50,51, // Letters 'n' through 'z' + -9,-9,-9,-9,-9 // Decimal 123 - 127 + ,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 128 - 139 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 140 - 152 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 153 - 165 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 166 - 178 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 179 - 191 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 192 - 204 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 205 - 217 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 218 - 230 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 231 - 243 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9 // Decimal 244 - 255 + }; + + +/* ******** U R L S A F E B A S E 6 4 A L P H A B E T ******** */ + + /** + * Used in the URL- and Filename-safe dialect described in Section 4 of RFC3548: + * http://www.faqs.org/rfcs/rfc3548.html. + * Notice that the last two bytes become "hyphen" and "underscore" instead of "plus" and "slash." + */ + private final static byte[] _URL_SAFE_ALPHABET = { + (byte)'A', (byte)'B', (byte)'C', (byte)'D', (byte)'E', (byte)'F', (byte)'G', + (byte)'H', (byte)'I', (byte)'J', (byte)'K', (byte)'L', (byte)'M', (byte)'N', + (byte)'O', (byte)'P', (byte)'Q', (byte)'R', (byte)'S', (byte)'T', (byte)'U', + (byte)'V', (byte)'W', (byte)'X', (byte)'Y', (byte)'Z', + (byte)'a', (byte)'b', (byte)'c', (byte)'d', (byte)'e', (byte)'f', (byte)'g', + (byte)'h', (byte)'i', (byte)'j', (byte)'k', (byte)'l', (byte)'m', (byte)'n', + (byte)'o', (byte)'p', (byte)'q', (byte)'r', (byte)'s', (byte)'t', (byte)'u', + (byte)'v', (byte)'w', (byte)'x', (byte)'y', (byte)'z', + (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', + (byte)'6', (byte)'7', (byte)'8', (byte)'9', (byte)'-', (byte)'_' + }; + + /** + * Used in decoding URL- and Filename-safe dialects of Base64. + */ + private final static byte[] _URL_SAFE_DECODABET = { + -9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 0 - 8 + -5,-5, // Whitespace: Tab and Linefeed + -9,-9, // Decimal 11 - 12 + -5, // Whitespace: Carriage Return + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 14 - 26 + -9,-9,-9,-9,-9, // Decimal 27 - 31 + -5, // Whitespace: Space + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 33 - 42 + -9, // Plus sign at decimal 43 + -9, // Decimal 44 + 62, // Minus sign at decimal 45 + -9, // Decimal 46 + -9, // Slash at decimal 47 + 52,53,54,55,56,57,58,59,60,61, // Numbers zero through nine + -9,-9,-9, // Decimal 58 - 60 + -1, // Equals sign at decimal 61 + -9,-9,-9, // Decimal 62 - 64 + 0,1,2,3,4,5,6,7,8,9,10,11,12,13, // Letters 'A' through 'N' + 14,15,16,17,18,19,20,21,22,23,24,25, // Letters 'O' through 'Z' + -9,-9,-9,-9, // Decimal 91 - 94 + 63, // Underscore at decimal 95 + -9, // Decimal 96 + 26,27,28,29,30,31,32,33,34,35,36,37,38, // Letters 'a' through 'm' + 39,40,41,42,43,44,45,46,47,48,49,50,51, // Letters 'n' through 'z' + -9,-9,-9,-9,-9 // Decimal 123 - 127 + ,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 128 - 139 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 140 - 152 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 153 - 165 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 166 - 178 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 179 - 191 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 192 - 204 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 205 - 217 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 218 - 230 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 231 - 243 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9 // Decimal 244 - 255 + }; + + + +/* ******** O R D E R E D B A S E 6 4 A L P H A B E T ******** */ + + /** + * I don't get the point of this technique, but someone requested it, + * and it is described here: + * http://www.faqs.org/qa/rfcc-1940.html. + */ + private final static byte[] _ORDERED_ALPHABET = { + (byte)'-', + (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', + (byte)'5', (byte)'6', (byte)'7', (byte)'8', (byte)'9', + (byte)'A', (byte)'B', (byte)'C', (byte)'D', (byte)'E', (byte)'F', (byte)'G', + (byte)'H', (byte)'I', (byte)'J', (byte)'K', (byte)'L', (byte)'M', (byte)'N', + (byte)'O', (byte)'P', (byte)'Q', (byte)'R', (byte)'S', (byte)'T', (byte)'U', + (byte)'V', (byte)'W', (byte)'X', (byte)'Y', (byte)'Z', + (byte)'_', + (byte)'a', (byte)'b', (byte)'c', (byte)'d', (byte)'e', (byte)'f', (byte)'g', + (byte)'h', (byte)'i', (byte)'j', (byte)'k', (byte)'l', (byte)'m', (byte)'n', + (byte)'o', (byte)'p', (byte)'q', (byte)'r', (byte)'s', (byte)'t', (byte)'u', + (byte)'v', (byte)'w', (byte)'x', (byte)'y', (byte)'z' + }; + + /** + * Used in decoding the "ordered" dialect of Base64. + */ + private final static byte[] _ORDERED_DECODABET = { + -9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 0 - 8 + -5,-5, // Whitespace: Tab and Linefeed + -9,-9, // Decimal 11 - 12 + -5, // Whitespace: Carriage Return + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 14 - 26 + -9,-9,-9,-9,-9, // Decimal 27 - 31 + -5, // Whitespace: Space + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 33 - 42 + -9, // Plus sign at decimal 43 + -9, // Decimal 44 + 0, // Minus sign at decimal 45 + -9, // Decimal 46 + -9, // Slash at decimal 47 + 1,2,3,4,5,6,7,8,9,10, // Numbers zero through nine + -9,-9,-9, // Decimal 58 - 60 + -1, // Equals sign at decimal 61 + -9,-9,-9, // Decimal 62 - 64 + 11,12,13,14,15,16,17,18,19,20,21,22,23, // Letters 'A' through 'M' + 24,25,26,27,28,29,30,31,32,33,34,35,36, // Letters 'N' through 'Z' + -9,-9,-9,-9, // Decimal 91 - 94 + 37, // Underscore at decimal 95 + -9, // Decimal 96 + 38,39,40,41,42,43,44,45,46,47,48,49,50, // Letters 'a' through 'm' + 51,52,53,54,55,56,57,58,59,60,61,62,63, // Letters 'n' through 'z' + -9,-9,-9,-9,-9 // Decimal 123 - 127 + ,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 128 - 139 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 140 - 152 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 153 - 165 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 166 - 178 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 179 - 191 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 192 - 204 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 205 - 217 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 218 - 230 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 231 - 243 + -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9 // Decimal 244 - 255 + }; + + +/* ******** D E T E R M I N E W H I C H A L H A B E T ******** */ + + + /** + * Returns one of the _SOMETHING_ALPHABET byte arrays depending on + * the options specified. + * It's possible, though silly, to specify ORDERED and URLSAFE + * in which case one of them will be picked, though there is + * no guarantee as to which one will be picked. + */ + private final static byte[] getAlphabet( int options ) { + if ((options & URL_SAFE) == URL_SAFE) { + return _URL_SAFE_ALPHABET; + } else if ((options & ORDERED) == ORDERED) { + return _ORDERED_ALPHABET; + } else { + return _STANDARD_ALPHABET; + } + } // end getAlphabet + + + /** + * Returns one of the _SOMETHING_DECODABET byte arrays depending on + * the options specified. + * It's possible, though silly, to specify ORDERED and URL_SAFE + * in which case one of them will be picked, though there is + * no guarantee as to which one will be picked. + */ + private final static byte[] getDecodabet( int options ) { + if( (options & URL_SAFE) == URL_SAFE) { + return _URL_SAFE_DECODABET; + } else if ((options & ORDERED) == ORDERED) { + return _ORDERED_DECODABET; + } else { + return _STANDARD_DECODABET; + } + } // end getAlphabet + + + + /** Defeats instantiation. */ + private Base64(){} + + + + +/* ******** E N C O D I N G M E T H O D S ******** */ + + + /** + * Encodes up to the first three bytes of array threeBytes + * and returns a four-byte array in Base64 notation. + * The actual number of significant bytes in your array is + * given by numSigBytes. + * The array threeBytes needs only be as big as + * numSigBytes. + * Code can reuse a byte array by passing a four-byte array as b4. + * + * @param b4 A reusable byte array to reduce array instantiation + * @param threeBytes the array to convert + * @param numSigBytes the number of significant bytes in your array + * @return four byte array in Base64 notation. + * @since 1.5.1 + */ + private static byte[] encode3to4( byte[] b4, byte[] threeBytes, int numSigBytes, int options ) { + encode3to4( threeBytes, 0, numSigBytes, b4, 0, options ); + return b4; + } // end encode3to4 + + + /** + *

Encodes up to three bytes of the array source + * and writes the resulting four Base64 bytes to destination. + * The source and destination arrays can be manipulated + * anywhere along their length by specifying + * srcOffset and destOffset. + * This method does not check to make sure your arrays + * are large enough to accomodate srcOffset + 3 for + * the source array or destOffset + 4 for + * the destination array. + * The actual number of significant bytes in your array is + * given by numSigBytes.

+ *

This is the lowest level of the encoding methods with + * all possible parameters.

+ * + * @param source the array to convert + * @param srcOffset the index where conversion begins + * @param numSigBytes the number of significant bytes in your array + * @param destination the array to hold the conversion + * @param destOffset the index where output will be put + * @return the destination array + * @since 1.3 + */ + private static byte[] encode3to4( + byte[] source, int srcOffset, int numSigBytes, + byte[] destination, int destOffset, int options ) { + + byte[] ALPHABET = getAlphabet( options ); + + // 1 2 3 + // 01234567890123456789012345678901 Bit position + // --------000000001111111122222222 Array position from threeBytes + // --------| || || || | Six bit groups to index ALPHABET + // >>18 >>12 >> 6 >> 0 Right shift necessary + // 0x3f 0x3f 0x3f Additional AND + + // Create buffer with zero-padding if there are only one or two + // significant bytes passed in the array. + // We have to shift left 24 in order to flush out the 1's that appear + // when Java treats a value as negative that is cast from a byte to an int. + int inBuff = ( numSigBytes > 0 ? ((source[ srcOffset ] << 24) >>> 8) : 0 ) + | ( numSigBytes > 1 ? ((source[ srcOffset + 1 ] << 24) >>> 16) : 0 ) + | ( numSigBytes > 2 ? ((source[ srcOffset + 2 ] << 24) >>> 24) : 0 ); + + switch( numSigBytes ) + { + case 3: + destination[ destOffset ] = ALPHABET[ (inBuff >>> 18) ]; + destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ]; + destination[ destOffset + 2 ] = ALPHABET[ (inBuff >>> 6) & 0x3f ]; + destination[ destOffset + 3 ] = ALPHABET[ (inBuff ) & 0x3f ]; + return destination; + + case 2: + destination[ destOffset ] = ALPHABET[ (inBuff >>> 18) ]; + destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ]; + destination[ destOffset + 2 ] = ALPHABET[ (inBuff >>> 6) & 0x3f ]; + destination[ destOffset + 3 ] = EQUALS_SIGN; + return destination; + + case 1: + destination[ destOffset ] = ALPHABET[ (inBuff >>> 18) ]; + destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ]; + destination[ destOffset + 2 ] = EQUALS_SIGN; + destination[ destOffset + 3 ] = EQUALS_SIGN; + return destination; + + default: + return destination; + } // end switch + } // end encode3to4 + + + + /** + * Performs Base64 encoding on the raw ByteBuffer, + * writing it to the encoded ByteBuffer. + * This is an experimental feature. Currently it does not + * pass along any options (such as {@link #DO_BREAK_LINES} + * or {@link #GZIP}. + * + * @param raw input buffer + * @param encoded output buffer + * @since 2.3 + */ + public static void encode( java.nio.ByteBuffer raw, java.nio.ByteBuffer encoded ){ + byte[] raw3 = new byte[3]; + byte[] enc4 = new byte[4]; + + while( raw.hasRemaining() ){ + int rem = Math.min(3,raw.remaining()); + raw.get(raw3,0,rem); + Base64.encode3to4(enc4, raw3, rem, Base64.NO_OPTIONS ); + encoded.put(enc4); + } // end input remaining + } + + + /** + * Performs Base64 encoding on the raw ByteBuffer, + * writing it to the encoded CharBuffer. + * This is an experimental feature. Currently it does not + * pass along any options (such as {@link #DO_BREAK_LINES} + * or {@link #GZIP}. + * + * @param raw input buffer + * @param encoded output buffer + * @since 2.3 + */ + public static void encode( java.nio.ByteBuffer raw, java.nio.CharBuffer encoded ){ + byte[] raw3 = new byte[3]; + byte[] enc4 = new byte[4]; + + while( raw.hasRemaining() ){ + int rem = Math.min(3,raw.remaining()); + raw.get(raw3,0,rem); + Base64.encode3to4(enc4, raw3, rem, Base64.NO_OPTIONS ); + for( int i = 0; i < 4; i++ ){ + encoded.put( (char)(enc4[i] & 0xFF) ); + } + } // end input remaining + } + + + + + /** + * Serializes an object and returns the Base64-encoded + * version of that serialized object. + * + *

As of v 2.3, if the object + * cannot be serialized or there is another error, + * the method will throw an java.io.IOException. This is new to v2.3! + * In earlier versions, it just returned a null value, but + * in retrospect that's a pretty poor way to handle it.

+ * + * The object is not GZip-compressed before being encoded. + * + * @param serializableObject The object to encode + * @return The Base64-encoded object + * @throws java.io.IOException if there is an error + * @throws NullPointerException if serializedObject is null + * @since 1.4 + */ + public static String encodeObject( java.io.Serializable serializableObject ) + throws java.io.IOException { + return encodeObject( serializableObject, NO_OPTIONS ); + } // end encodeObject + + + + /** + * Serializes an object and returns the Base64-encoded + * version of that serialized object. + * + *

As of v 2.3, if the object + * cannot be serialized or there is another error, + * the method will throw an java.io.IOException. This is new to v2.3! + * In earlier versions, it just returned a null value, but + * in retrospect that's a pretty poor way to handle it.

+ * + * The object is not GZip-compressed before being encoded. + *

+ * Example options:

+     *   GZIP: gzip-compresses object before encoding it.
+     *   DO_BREAK_LINES: break lines at 76 characters
+     *
+ *

+ * Example: encodeObject( myObj, Base64.GZIP ) or + *

+ * Example: encodeObject( myObj, Base64.GZIP | Base64.DO_BREAK_LINES ) + * + * @param serializableObject The object to encode + * @param options Specified options + * @return The Base64-encoded object + * @see Base64#GZIP + * @see Base64#DO_BREAK_LINES + * @throws java.io.IOException if there is an error + * @since 2.0 + */ + public static String encodeObject( java.io.Serializable serializableObject, int options ) + throws java.io.IOException { + + if( serializableObject == null ){ + throw new NullPointerException( "Cannot serialize a null object." ); + } // end if: null + + // Streams + java.io.ByteArrayOutputStream baos = null; + java.io.OutputStream b64os = null; + java.util.zip.GZIPOutputStream gzos = null; + java.io.ObjectOutputStream oos = null; + + + try { + // ObjectOutputStream -> (GZIP) -> Base64 -> ByteArrayOutputStream + baos = new java.io.ByteArrayOutputStream(); + b64os = new Base64.OutputStream( baos, ENCODE | options ); + if( (options & GZIP) != 0 ){ + // Gzip + gzos = new java.util.zip.GZIPOutputStream(b64os); + oos = new java.io.ObjectOutputStream( gzos ); + } else { + // Not gzipped + oos = new java.io.ObjectOutputStream( b64os ); + } + oos.writeObject( serializableObject ); + } // end try + catch( java.io.IOException e ) { + // Catch it and then throw it immediately so that + // the finally{} block is called for cleanup. + throw e; + } // end catch + finally { + try{ oos.close(); } catch( Exception e ){} + try{ gzos.close(); } catch( Exception e ){} + try{ b64os.close(); } catch( Exception e ){} + try{ baos.close(); } catch( Exception e ){} + } // end finally + + // Return value according to relevant encoding. + try { + return new String( baos.toByteArray(), PREFERRED_ENCODING ); + } // end try + catch (java.io.UnsupportedEncodingException uue){ + // Fall back to some Java default + return new String( baos.toByteArray() ); + } // end catch + + } // end encode + + + + /** + * Encodes a byte array into Base64 notation. + * Does not GZip-compress data. + * + * @param source The data to convert + * @return The data in Base64-encoded form + * @throws NullPointerException if source array is null + * @since 1.4 + */ + public static String encodeBytes( byte[] source ) { + // Since we're not going to have the GZIP encoding turned on, + // we're not going to have an java.io.IOException thrown, so + // we should not force the user to have to catch it. + String encoded = null; + try { + encoded = encodeBytes(source, 0, source.length, NO_OPTIONS); + } catch (java.io.IOException ex) { + assert false : ex.getMessage(); + } // end catch + assert encoded != null; + return encoded; + } // end encodeBytes + + + + /** + * Encodes a byte array into Base64 notation. + *

+ * Example options:

+     *   GZIP: gzip-compresses object before encoding it.
+     *   DO_BREAK_LINES: break lines at 76 characters
+     *     Note: Technically, this makes your encoding non-compliant.
+     *
+ *

+ * Example: encodeBytes( myData, Base64.GZIP ) or + *

+ * Example: encodeBytes( myData, Base64.GZIP | Base64.DO_BREAK_LINES ) + * + * + *

As of v 2.3, if there is an error with the GZIP stream, + * the method will throw an java.io.IOException. This is new to v2.3! + * In earlier versions, it just returned a null value, but + * in retrospect that's a pretty poor way to handle it.

+ * + * + * @param source The data to convert + * @param options Specified options + * @return The Base64-encoded data as a String + * @see Base64#GZIP + * @see Base64#DO_BREAK_LINES + * @throws java.io.IOException if there is an error + * @throws NullPointerException if source array is null + * @since 2.0 + */ + public static String encodeBytes( byte[] source, int options ) throws java.io.IOException { + return encodeBytes( source, 0, source.length, options ); + } // end encodeBytes + + + /** + * Encodes a byte array into Base64 notation. + * Does not GZip-compress data. + * + *

As of v 2.3, if there is an error, + * the method will throw an java.io.IOException. This is new to v2.3! + * In earlier versions, it just returned a null value, but + * in retrospect that's a pretty poor way to handle it.

+ * + * + * @param source The data to convert + * @param off Offset in array where conversion should begin + * @param len Length of data to convert + * @return The Base64-encoded data as a String + * @throws NullPointerException if source array is null + * @throws IllegalArgumentException if source array, offset, or length are invalid + * @since 1.4 + */ + public static String encodeBytes( byte[] source, int off, int len ) { + // Since we're not going to have the GZIP encoding turned on, + // we're not going to have an java.io.IOException thrown, so + // we should not force the user to have to catch it. + String encoded = null; + try { + encoded = encodeBytes( source, off, len, NO_OPTIONS ); + } catch (java.io.IOException ex) { + assert false : ex.getMessage(); + } // end catch + assert encoded != null; + return encoded; + } // end encodeBytes + + + + /** + * Encodes a byte array into Base64 notation. + *

+ * Example options:

+     *   GZIP: gzip-compresses object before encoding it.
+     *   DO_BREAK_LINES: break lines at 76 characters
+     *     Note: Technically, this makes your encoding non-compliant.
+     *
+ *

+ * Example: encodeBytes( myData, Base64.GZIP ) or + *

+ * Example: encodeBytes( myData, Base64.GZIP | Base64.DO_BREAK_LINES ) + * + * + *

As of v 2.3, if there is an error with the GZIP stream, + * the method will throw an java.io.IOException. This is new to v2.3! + * In earlier versions, it just returned a null value, but + * in retrospect that's a pretty poor way to handle it.

+ * + * + * @param source The data to convert + * @param off Offset in array where conversion should begin + * @param len Length of data to convert + * @param options Specified options + * @return The Base64-encoded data as a String + * @see Base64#GZIP + * @see Base64#DO_BREAK_LINES + * @throws java.io.IOException if there is an error + * @throws NullPointerException if source array is null + * @throws IllegalArgumentException if source array, offset, or length are invalid + * @since 2.0 + */ + public static String encodeBytes( byte[] source, int off, int len, int options ) throws java.io.IOException { + byte[] encoded = encodeBytesToBytes( source, off, len, options ); + + // Return value according to relevant encoding. + try { + return new String( encoded, PREFERRED_ENCODING ); + } // end try + catch (java.io.UnsupportedEncodingException uue) { + return new String( encoded ); + } // end catch + + } // end encodeBytes + + + + + /** + * Similar to {@link #encodeBytes(byte[])} but returns + * a byte array instead of instantiating a String. This is more efficient + * if you're working with I/O streams and have large data sets to encode. + * + * + * @param source The data to convert + * @return The Base64-encoded data as a byte[] (of ASCII characters) + * @throws NullPointerException if source array is null + * @since 2.3.1 + */ + public static byte[] encodeBytesToBytes( byte[] source ) { + byte[] encoded = null; + try { + encoded = encodeBytesToBytes( source, 0, source.length, Base64.NO_OPTIONS ); + } catch( java.io.IOException ex ) { + assert false : "IOExceptions only come from GZipping, which is turned off: " + ex.getMessage(); + } + return encoded; + } + + + /** + * Similar to {@link #encodeBytes(byte[], int, int, int)} but returns + * a byte array instead of instantiating a String. This is more efficient + * if you're working with I/O streams and have large data sets to encode. + * + * + * @param source The data to convert + * @param off Offset in array where conversion should begin + * @param len Length of data to convert + * @param options Specified options + * @return The Base64-encoded data as a String + * @see Base64#GZIP + * @see Base64#DO_BREAK_LINES + * @throws java.io.IOException if there is an error + * @throws NullPointerException if source array is null + * @throws IllegalArgumentException if source array, offset, or length are invalid + * @since 2.3.1 + */ + public static byte[] encodeBytesToBytes( byte[] source, int off, int len, int options ) throws java.io.IOException { + + if( source == null ){ + throw new NullPointerException( "Cannot serialize a null array." ); + } // end if: null + + if( off < 0 ){ + throw new IllegalArgumentException( "Cannot have negative offset: " + off ); + } // end if: off < 0 + + if( len < 0 ){ + throw new IllegalArgumentException( "Cannot have length offset: " + len ); + } // end if: len < 0 + + if( off + len > source.length ){ + throw new IllegalArgumentException( + String.format( "Cannot have offset of %d and length of %d with array of length %d", off,len,source.length)); + } // end if: off < 0 + + + + // Compress? + if( (options & GZIP) != 0 ) { + java.io.ByteArrayOutputStream baos = null; + java.util.zip.GZIPOutputStream gzos = null; + Base64.OutputStream b64os = null; + + try { + // GZip -> Base64 -> ByteArray + baos = new java.io.ByteArrayOutputStream(); + b64os = new Base64.OutputStream( baos, ENCODE | options ); + gzos = new java.util.zip.GZIPOutputStream( b64os ); + + gzos.write( source, off, len ); + gzos.close(); + } // end try + catch( java.io.IOException e ) { + // Catch it and then throw it immediately so that + // the finally{} block is called for cleanup. + throw e; + } // end catch + finally { + try{ gzos.close(); } catch( Exception e ){} + try{ b64os.close(); } catch( Exception e ){} + try{ baos.close(); } catch( Exception e ){} + } // end finally + + return baos.toByteArray(); + } // end if: compress + + // Else, don't compress. Better not to use streams at all then. + else { + boolean breakLines = (options & DO_BREAK_LINES) != 0; + + //int len43 = len * 4 / 3; + //byte[] outBuff = new byte[ ( len43 ) // Main 4:3 + // + ( (len % 3) > 0 ? 4 : 0 ) // Account for padding + // + (breakLines ? ( len43 / MAX_LINE_LENGTH ) : 0) ]; // New lines + // Try to determine more precisely how big the array needs to be. + // If we get it right, we don't have to do an array copy, and + // we save a bunch of memory. + int encLen = ( len / 3 ) * 4 + ( len % 3 > 0 ? 4 : 0 ); // Bytes needed for actual encoding + if( breakLines ){ + encLen += encLen / MAX_LINE_LENGTH; // Plus extra newline characters + } + byte[] outBuff = new byte[ encLen ]; + + + int d = 0; + int e = 0; + int len2 = len - 2; + int lineLength = 0; + for( ; d < len2; d+=3, e+=4 ) { + encode3to4( source, d+off, 3, outBuff, e, options ); + + lineLength += 4; + if( breakLines && lineLength >= MAX_LINE_LENGTH ) + { + outBuff[e+4] = NEW_LINE; + e++; + lineLength = 0; + } // end if: end of line + } // en dfor: each piece of array + + if( d < len ) { + encode3to4( source, d+off, len - d, outBuff, e, options ); + e += 4; + } // end if: some padding needed + + + // Only resize array if we didn't guess it right. + if( e <= outBuff.length - 1 ){ + // If breaking lines and the last byte falls right at + // the line length (76 bytes per line), there will be + // one extra byte, and the array will need to be resized. + // Not too bad of an estimate on array size, I'd say. + byte[] finalOut = new byte[e]; + System.arraycopy(outBuff,0, finalOut,0,e); + //System.err.println("Having to resize array from " + outBuff.length + " to " + e ); + return finalOut; + } else { + //System.err.println("No need to resize array."); + return outBuff; + } + + } // end else: don't compress + + } // end encodeBytesToBytes + + + + + +/* ******** D E C O D I N G M E T H O D S ******** */ + + + /** + * Decodes four bytes from array source + * and writes the resulting bytes (up to three of them) + * to destination. + * The source and destination arrays can be manipulated + * anywhere along their length by specifying + * srcOffset and destOffset. + * This method does not check to make sure your arrays + * are large enough to accomodate srcOffset + 4 for + * the source array or destOffset + 3 for + * the destination array. + * This method returns the actual number of bytes that + * were converted from the Base64 encoding. + *

This is the lowest level of the decoding methods with + * all possible parameters.

+ * + * + * @param source the array to convert + * @param srcOffset the index where conversion begins + * @param destination the array to hold the conversion + * @param destOffset the index where output will be put + * @param options alphabet type is pulled from this (standard, url-safe, ordered) + * @return the number of decoded bytes converted + * @throws NullPointerException if source or destination arrays are null + * @throws IllegalArgumentException if srcOffset or destOffset are invalid + * or there is not enough room in the array. + * @since 1.3 + */ + private static int decode4to3( + byte[] source, int srcOffset, + byte[] destination, int destOffset, int options ) { + + // Lots of error checking and exception throwing + if( source == null ){ + throw new NullPointerException( "Source array was null." ); + } // end if + if( destination == null ){ + throw new NullPointerException( "Destination array was null." ); + } // end if + if( srcOffset < 0 || srcOffset + 3 >= source.length ){ + throw new IllegalArgumentException( String.format( + "Source array with length %d cannot have offset of %d and still process four bytes.", source.length, srcOffset ) ); + } // end if + if( destOffset < 0 || destOffset +2 >= destination.length ){ + throw new IllegalArgumentException( String.format( + "Destination array with length %d cannot have offset of %d and still store three bytes.", destination.length, destOffset ) ); + } // end if + + + byte[] DECODABET = getDecodabet( options ); + + // Example: Dk== + if( source[ srcOffset + 2] == EQUALS_SIGN ) { + // Two ways to do the same thing. Don't know which way I like best. + //int outBuff = ( ( DECODABET[ source[ srcOffset ] ] << 24 ) >>> 6 ) + // | ( ( DECODABET[ source[ srcOffset + 1] ] << 24 ) >>> 12 ); + int outBuff = ( ( DECODABET[ source[ srcOffset ] ] & 0xFF ) << 18 ) + | ( ( DECODABET[ source[ srcOffset + 1] ] & 0xFF ) << 12 ); + + destination[ destOffset ] = (byte)( outBuff >>> 16 ); + return 1; + } + + // Example: DkL= + else if( source[ srcOffset + 3 ] == EQUALS_SIGN ) { + // Two ways to do the same thing. Don't know which way I like best. + //int outBuff = ( ( DECODABET[ source[ srcOffset ] ] << 24 ) >>> 6 ) + // | ( ( DECODABET[ source[ srcOffset + 1 ] ] << 24 ) >>> 12 ) + // | ( ( DECODABET[ source[ srcOffset + 2 ] ] << 24 ) >>> 18 ); + int outBuff = ( ( DECODABET[ source[ srcOffset ] ] & 0xFF ) << 18 ) + | ( ( DECODABET[ source[ srcOffset + 1 ] ] & 0xFF ) << 12 ) + | ( ( DECODABET[ source[ srcOffset + 2 ] ] & 0xFF ) << 6 ); + + destination[ destOffset ] = (byte)( outBuff >>> 16 ); + destination[ destOffset + 1 ] = (byte)( outBuff >>> 8 ); + return 2; + } + + // Example: DkLE + else { + // Two ways to do the same thing. Don't know which way I like best. + //int outBuff = ( ( DECODABET[ source[ srcOffset ] ] << 24 ) >>> 6 ) + // | ( ( DECODABET[ source[ srcOffset + 1 ] ] << 24 ) >>> 12 ) + // | ( ( DECODABET[ source[ srcOffset + 2 ] ] << 24 ) >>> 18 ) + // | ( ( DECODABET[ source[ srcOffset + 3 ] ] << 24 ) >>> 24 ); + int outBuff = ( ( DECODABET[ source[ srcOffset ] ] & 0xFF ) << 18 ) + | ( ( DECODABET[ source[ srcOffset + 1 ] ] & 0xFF ) << 12 ) + | ( ( DECODABET[ source[ srcOffset + 2 ] ] & 0xFF ) << 6) + | ( ( DECODABET[ source[ srcOffset + 3 ] ] & 0xFF ) ); + + + destination[ destOffset ] = (byte)( outBuff >> 16 ); + destination[ destOffset + 1 ] = (byte)( outBuff >> 8 ); + destination[ destOffset + 2 ] = (byte)( outBuff ); + + return 3; + } + } // end decodeToBytes + + + + + + /** + * Low-level access to decoding ASCII characters in + * the form of a byte array. Ignores GUNZIP option, if + * it's set. This is not generally a recommended method, + * although it is used internally as part of the decoding process. + * Special case: if len = 0, an empty array is returned. Still, + * if you need more speed and reduced memory footprint (and aren't + * gzipping), consider this method. + * + * @param source The Base64 encoded data + * @return decoded data + * @since 2.3.1 + */ + public static byte[] decode( byte[] source ) + throws java.io.IOException { + byte[] decoded = null; +// try { + decoded = decode( source, 0, source.length, Base64.NO_OPTIONS ); +// } catch( java.io.IOException ex ) { +// assert false : "IOExceptions only come from GZipping, which is turned off: " + ex.getMessage(); +// } + return decoded; + } + + + + /** + * Low-level access to decoding ASCII characters in + * the form of a byte array. Ignores GUNZIP option, if + * it's set. This is not generally a recommended method, + * although it is used internally as part of the decoding process. + * Special case: if len = 0, an empty array is returned. Still, + * if you need more speed and reduced memory footprint (and aren't + * gzipping), consider this method. + * + * @param source The Base64 encoded data + * @param off The offset of where to begin decoding + * @param len The length of characters to decode + * @param options Can specify options such as alphabet type to use + * @return decoded data + * @throws java.io.IOException If bogus characters exist in source data + * @since 1.3 + */ + public static byte[] decode( byte[] source, int off, int len, int options ) + throws java.io.IOException { + + // Lots of error checking and exception throwing + if( source == null ){ + throw new NullPointerException( "Cannot decode null source array." ); + } // end if + if( off < 0 || off + len > source.length ){ + throw new IllegalArgumentException( String.format( + "Source array with length %d cannot have offset of %d and process %d bytes.", source.length, off, len ) ); + } // end if + + if( len == 0 ){ + return new byte[0]; + }else if( len < 4 ){ + throw new IllegalArgumentException( + "Base64-encoded string must have at least four characters, but length specified was " + len ); + } // end if + + byte[] DECODABET = getDecodabet( options ); + + int len34 = len * 3 / 4; // Estimate on array size + byte[] outBuff = new byte[ len34 ]; // Upper limit on size of output + int outBuffPosn = 0; // Keep track of where we're writing + + byte[] b4 = new byte[4]; // Four byte buffer from source, eliminating white space + int b4Posn = 0; // Keep track of four byte input buffer + int i = 0; // Source array counter + byte sbiDecode = 0; // Special value from DECODABET + + for( i = off; i < off+len; i++ ) { // Loop through source + + sbiDecode = DECODABET[ source[i]&0xFF ]; + + // White space, Equals sign, or legit Base64 character + // Note the values such as -5 and -9 in the + // DECODABETs at the top of the file. + if( sbiDecode >= WHITE_SPACE_ENC ) { + if( sbiDecode >= EQUALS_SIGN_ENC ) { + b4[ b4Posn++ ] = source[i]; // Save non-whitespace + if( b4Posn > 3 ) { // Time to decode? + outBuffPosn += decode4to3( b4, 0, outBuff, outBuffPosn, options ); + b4Posn = 0; + + // If that was the equals sign, break out of 'for' loop + if( source[i] == EQUALS_SIGN ) { + break; + } // end if: equals sign + } // end if: quartet built + } // end if: equals sign or better + } // end if: white space, equals sign or better + else { + // There's a bad input character in the Base64 stream. + throw new java.io.IOException( String.format( + "Bad Base64 input character decimal %d in array position %d", ((int)source[i])&0xFF, i ) ); + } // end else: + } // each input character + + byte[] out = new byte[ outBuffPosn ]; + System.arraycopy( outBuff, 0, out, 0, outBuffPosn ); + return out; + } // end decode + + + + + /** + * Decodes data from Base64 notation, automatically + * detecting gzip-compressed data and decompressing it. + * + * @param s the string to decode + * @return the decoded data + * @throws java.io.IOException If there is a problem + * @since 1.4 + */ + public static byte[] decode( String s ) throws java.io.IOException { + return decode( s, NO_OPTIONS ); + } + + + + /** + * Decodes data from Base64 notation, automatically + * detecting gzip-compressed data and decompressing it. + * + * @param s the string to decode + * @param options encode options such as URL_SAFE + * @return the decoded data + * @throws java.io.IOException if there is an error + * @throws NullPointerException if s is null + * @since 1.4 + */ + public static byte[] decode( String s, int options ) throws java.io.IOException { + + if( s == null ){ + throw new NullPointerException( "Input string was null." ); + } // end if + + byte[] bytes; + try { + bytes = s.getBytes( PREFERRED_ENCODING ); + } // end try + catch( java.io.UnsupportedEncodingException uee ) { + bytes = s.getBytes(); + } // end catch + // + + // Decode + bytes = decode( bytes, 0, bytes.length, options ); + + // Check to see if it's gzip-compressed + // GZIP Magic Two-Byte Number: 0x8b1f (35615) + boolean dontGunzip = (options & DONT_GUNZIP) != 0; + if( (bytes != null) && (bytes.length >= 4) && (!dontGunzip) ) { + + int head = ((int)bytes[0] & 0xff) | ((bytes[1] << 8) & 0xff00); + if( java.util.zip.GZIPInputStream.GZIP_MAGIC == head ) { + java.io.ByteArrayInputStream bais = null; + java.util.zip.GZIPInputStream gzis = null; + java.io.ByteArrayOutputStream baos = null; + byte[] buffer = new byte[2048]; + int length = 0; + + try { + baos = new java.io.ByteArrayOutputStream(); + bais = new java.io.ByteArrayInputStream( bytes ); + gzis = new java.util.zip.GZIPInputStream( bais ); + + while( ( length = gzis.read( buffer ) ) >= 0 ) { + baos.write(buffer,0,length); + } // end while: reading input + + // No error? Get new bytes. + bytes = baos.toByteArray(); + + } // end try + catch( java.io.IOException e ) { + e.printStackTrace(); + // Just return originally-decoded bytes + } // end catch + finally { + try{ baos.close(); } catch( Exception e ){} + try{ gzis.close(); } catch( Exception e ){} + try{ bais.close(); } catch( Exception e ){} + } // end finally + + } // end if: gzipped + } // end if: bytes.length >= 2 + + return bytes; + } // end decode + + + + /** + * Attempts to decode Base64 data and deserialize a Java + * Object within. Returns null if there was an error. + * + * @param encodedObject The Base64 data to decode + * @return The decoded and deserialized object + * @throws NullPointerException if encodedObject is null + * @throws java.io.IOException if there is a general error + * @throws ClassNotFoundException if the decoded object is of a + * class that cannot be found by the JVM + * @since 1.5 + */ + public static Object decodeToObject( String encodedObject ) + throws java.io.IOException, java.lang.ClassNotFoundException { + return decodeToObject(encodedObject,NO_OPTIONS,null); + } + + + /** + * Attempts to decode Base64 data and deserialize a Java + * Object within. Returns null if there was an error. + * If loader is not null, it will be the class loader + * used when deserializing. + * + * @param encodedObject The Base64 data to decode + * @param options Various parameters related to decoding + * @param loader Optional class loader to use in deserializing classes. + * @return The decoded and deserialized object + * @throws NullPointerException if encodedObject is null + * @throws java.io.IOException if there is a general error + * @throws ClassNotFoundException if the decoded object is of a + * class that cannot be found by the JVM + * @since 2.3.4 + */ + public static Object decodeToObject( + String encodedObject, int options, final ClassLoader loader ) + throws java.io.IOException, java.lang.ClassNotFoundException { + + // Decode and gunzip if necessary + byte[] objBytes = decode( encodedObject, options ); + + java.io.ByteArrayInputStream bais = null; + java.io.ObjectInputStream ois = null; + Object obj = null; + + try { + bais = new java.io.ByteArrayInputStream( objBytes ); + + // If no custom class loader is provided, use Java's builtin OIS. + if( loader == null ){ + ois = new java.io.ObjectInputStream( bais ); + } // end if: no loader provided + + // Else make a customized object input stream that uses + // the provided class loader. + else { + ois = new java.io.ObjectInputStream(bais){ + @Override + public Class resolveClass(java.io.ObjectStreamClass streamClass) + throws java.io.IOException, ClassNotFoundException { + Class c = Class.forName(streamClass.getName(), false, loader); + if( c == null ){ + return super.resolveClass(streamClass); + } else { + return c; // Class loader knows of this class. + } // end else: not null + } // end resolveClass + }; // end ois + } // end else: no custom class loader + + obj = ois.readObject(); + } // end try + catch( java.io.IOException e ) { + throw e; // Catch and throw in order to execute finally{} + } // end catch + catch( java.lang.ClassNotFoundException e ) { + throw e; // Catch and throw in order to execute finally{} + } // end catch + finally { + try{ bais.close(); } catch( Exception e ){} + try{ ois.close(); } catch( Exception e ){} + } // end finally + + return obj; + } // end decodeObject + + + + /** + * Convenience method for encoding data to a file. + * + *

As of v 2.3, if there is a error, + * the method will throw an java.io.IOException. This is new to v2.3! + * In earlier versions, it just returned false, but + * in retrospect that's a pretty poor way to handle it.

+ * + * @param dataToEncode byte array of data to encode in base64 form + * @param filename Filename for saving encoded data + * @throws java.io.IOException if there is an error + * @throws NullPointerException if dataToEncode is null + * @since 2.1 + */ + public static void encodeToFile( byte[] dataToEncode, String filename ) + throws java.io.IOException { + + if( dataToEncode == null ){ + throw new NullPointerException( "Data to encode was null." ); + } // end iff + + Base64.OutputStream bos = null; + try { + bos = new Base64.OutputStream( + new java.io.FileOutputStream( filename ), Base64.ENCODE ); + bos.write( dataToEncode ); + } // end try + catch( java.io.IOException e ) { + throw e; // Catch and throw to execute finally{} block + } // end catch: java.io.IOException + finally { + try{ bos.close(); } catch( Exception e ){} + } // end finally + + } // end encodeToFile + + + /** + * Convenience method for decoding data to a file. + * + *

As of v 2.3, if there is a error, + * the method will throw an java.io.IOException. This is new to v2.3! + * In earlier versions, it just returned false, but + * in retrospect that's a pretty poor way to handle it.

+ * + * @param dataToDecode Base64-encoded data as a string + * @param filename Filename for saving decoded data + * @throws java.io.IOException if there is an error + * @since 2.1 + */ + public static void decodeToFile( String dataToDecode, String filename ) + throws java.io.IOException { + + Base64.OutputStream bos = null; + try{ + bos = new Base64.OutputStream( + new java.io.FileOutputStream( filename ), Base64.DECODE ); + bos.write( dataToDecode.getBytes( PREFERRED_ENCODING ) ); + } // end try + catch( java.io.IOException e ) { + throw e; // Catch and throw to execute finally{} block + } // end catch: java.io.IOException + finally { + try{ bos.close(); } catch( Exception e ){} + } // end finally + + } // end decodeToFile + + + + + /** + * Convenience method for reading a base64-encoded + * file and decoding it. + * + *

As of v 2.3, if there is a error, + * the method will throw an java.io.IOException. This is new to v2.3! + * In earlier versions, it just returned false, but + * in retrospect that's a pretty poor way to handle it.

+ * + * @param filename Filename for reading encoded data + * @return decoded byte array + * @throws java.io.IOException if there is an error + * @since 2.1 + */ + public static byte[] decodeFromFile( String filename ) + throws java.io.IOException { + + byte[] decodedData = null; + Base64.InputStream bis = null; + try + { + // Set up some useful variables + java.io.File file = new java.io.File( filename ); + byte[] buffer = null; + int length = 0; + int numBytes = 0; + + // Check for size of file + if( file.length() > Integer.MAX_VALUE ) + { + throw new java.io.IOException( "File is too big for this convenience method (" + file.length() + " bytes)." ); + } // end if: file too big for int index + buffer = new byte[ (int)file.length() ]; + + // Open a stream + bis = new Base64.InputStream( + new java.io.BufferedInputStream( + new java.io.FileInputStream( file ) ), Base64.DECODE ); + + // Read until done + while( ( numBytes = bis.read( buffer, length, 4096 ) ) >= 0 ) { + length += numBytes; + } // end while + + // Save in a variable to return + decodedData = new byte[ length ]; + System.arraycopy( buffer, 0, decodedData, 0, length ); + + } // end try + catch( java.io.IOException e ) { + throw e; // Catch and release to execute finally{} + } // end catch: java.io.IOException + finally { + try{ bis.close(); } catch( Exception e) {} + } // end finally + + return decodedData; + } // end decodeFromFile + + + /** + * See {@link #encodeFromFile(java.io.File)}. + * + * @param filename Filename for reading binary data + * @return base64-encoded string + * @throws java.io.IOException if there is an error + * @since 2.1 + */ + public static String encodeFromFile(String filename) throws java.io.IOException { + return encodeFromFile(new java.io.File(filename)); + } + + public static String readB64(File dir, String name) throws IOException { + return encodeFromFile(new File(dir, name)); + } + + /** + * Convenience method for reading a binary file + * and base64-encoding it. + * + *

As of v 2.3, if there is a error, + * the method will throw an java.io.IOException. This is new to v2.3! + * In earlier versions, it just returned false, but + * in retrospect that's a pretty poor way to handle it.

+ * + * @param file File for reading binary data + * @return base64-encoded string + * @throws java.io.IOException if there is an error + */ + public static String encodeFromFile(java.io.File file) throws java.io.IOException { + + String encodedData = null; + Base64.InputStream bis = null; + try + { + // Set up some useful variables + byte[] buffer = new byte[ Math.max((int)(file.length() * 1.4+1),40) ]; // Need max() for math on small files (v2.2.1); Need +1 for a few corner cases (v2.3.5) + int length = 0; + int numBytes = 0; + + // Open a stream + bis = new Base64.InputStream( + new java.io.BufferedInputStream( + new java.io.FileInputStream( file ) ), Base64.ENCODE ); + + // Read until done + while( ( numBytes = bis.read( buffer, length, 4096 ) ) >= 0 ) { + length += numBytes; + } // end while + + // Save in a variable to return + encodedData = new String( buffer, 0, length, Base64.PREFERRED_ENCODING ); + + } // end try + catch( java.io.IOException e ) { + throw e; // Catch and release to execute finally{} + } // end catch: java.io.IOException + finally { + try{ bis.close(); } catch( Exception e) {} + } // end finally + + return encodedData; + } // end encodeFromFile + + /** + * Reads infile and encodes it to outfile. + * + * @param infile Input file + * @param outfile Output file + * @throws java.io.IOException if there is an error + * @since 2.2 + */ + public static void encodeFileToFile( String infile, String outfile ) + throws java.io.IOException { + + String encoded = Base64.encodeFromFile( infile ); + java.io.OutputStream out = null; + try{ + out = new java.io.BufferedOutputStream( + new java.io.FileOutputStream( outfile ) ); + out.write( encoded.getBytes("US-ASCII") ); // Strict, 7-bit output. + } // end try + catch( java.io.IOException e ) { + throw e; // Catch and release to execute finally{} + } // end catch + finally { + try { out.close(); } + catch( Exception ex ){} + } // end finally + } // end encodeFileToFile + + + /** + * Reads infile and decodes it to outfile. + * + * @param infile Input file + * @param outfile Output file + * @throws java.io.IOException if there is an error + * @since 2.2 + */ + public static void decodeFileToFile( String infile, String outfile ) + throws java.io.IOException { + + byte[] decoded = Base64.decodeFromFile( infile ); + java.io.OutputStream out = null; + try{ + out = new java.io.BufferedOutputStream( + new java.io.FileOutputStream( outfile ) ); + out.write( decoded ); + } // end try + catch( java.io.IOException e ) { + throw e; // Catch and release to execute finally{} + } // end catch + finally { + try { out.close(); } + catch( Exception ex ){} + } // end finally + } // end decodeFileToFile + + + /* ******** I N N E R C L A S S I N P U T S T R E A M ******** */ + + + + /** + * A {@link Base64.InputStream} will read data from another + * java.io.InputStream, given in the constructor, + * and encode/decode to/from Base64 notation on the fly. + * + * @see Base64 + * @since 1.3 + */ + public static class InputStream extends java.io.FilterInputStream { + + private boolean encode; // Encoding or decoding + private int position; // Current position in the buffer + private byte[] buffer; // Small buffer holding converted data + private int bufferLength; // Length of buffer (3 or 4) + private int numSigBytes; // Number of meaningful bytes in the buffer + private int lineLength; + private boolean breakLines; // Break lines at less than 80 characters + private int options; // Record options used to create the stream. + private byte[] decodabet; // Local copies to avoid extra method calls + + + /** + * Constructs a {@link Base64.InputStream} in DECODE mode. + * + * @param in the java.io.InputStream from which to read data. + * @since 1.3 + */ + public InputStream( java.io.InputStream in ) { + this( in, DECODE ); + } // end constructor + + + /** + * Constructs a {@link Base64.InputStream} in + * either ENCODE or DECODE mode. + *

+ * Valid options:

+         *   ENCODE or DECODE: Encode or Decode as data is read.
+         *   DO_BREAK_LINES: break lines at 76 characters
+         *     (only meaningful when encoding)
+         *
+ *

+ * Example: new Base64.InputStream( in, Base64.DECODE ) + * + * + * @param in the java.io.InputStream from which to read data. + * @param options Specified options + * @see Base64#ENCODE + * @see Base64#DECODE + * @see Base64#DO_BREAK_LINES + * @since 2.0 + */ + public InputStream( java.io.InputStream in, int options ) { + + super( in ); + this.options = options; // Record for later + this.breakLines = (options & DO_BREAK_LINES) > 0; + this.encode = (options & ENCODE) > 0; + this.bufferLength = encode ? 4 : 3; + this.buffer = new byte[ bufferLength ]; + this.position = -1; + this.lineLength = 0; + this.decodabet = getDecodabet(options); + } // end constructor + + /** + * Reads enough of the input stream to convert + * to/from Base64 and returns the next byte. + * + * @return next byte + * @since 1.3 + */ + @Override + public int read() throws java.io.IOException { + + // Do we need to get data? + if( position < 0 ) { + if( encode ) { + byte[] b3 = new byte[3]; + int numBinaryBytes = 0; + for( int i = 0; i < 3; i++ ) { + int b = in.read(); + + // If end of stream, b is -1. + if( b >= 0 ) { + b3[i] = (byte)b; + numBinaryBytes++; + } else { + break; // out of for loop + } // end else: end of stream + + } // end for: each needed input byte + + if( numBinaryBytes > 0 ) { + encode3to4( b3, 0, numBinaryBytes, buffer, 0, options ); + position = 0; + numSigBytes = 4; + } // end if: got data + else { + return -1; // Must be end of stream + } // end else + } // end if: encoding + + // Else decoding + else { + byte[] b4 = new byte[4]; + int i = 0; + for( i = 0; i < 4; i++ ) { + // Read four "meaningful" bytes: + int b = 0; + do{ b = in.read(); } + while( b >= 0 && decodabet[ b & 0x7f ] <= WHITE_SPACE_ENC ); + + if( b < 0 ) { + break; // Reads a -1 if end of stream + } // end if: end of stream + + b4[i] = (byte)b; + } // end for: each needed input byte + + if( i == 4 ) { + numSigBytes = decode4to3( b4, 0, buffer, 0, options ); + position = 0; + } // end if: got four characters + else if( i == 0 ){ + return -1; + } // end else if: also padded correctly + else { + // Must have broken out from above. + throw new java.io.IOException( "Improperly padded Base64 input." ); + } // end + + } // end else: decode + } // end else: get data + + // Got data? + if( position >= 0 ) { + // End of relevant data? + if( /*!encode &&*/ position >= numSigBytes ){ + return -1; + } // end if: got data + + if( encode && breakLines && lineLength >= MAX_LINE_LENGTH ) { + lineLength = 0; + return '\n'; + } // end if + else { + lineLength++; // This isn't important when decoding + // but throwing an extra "if" seems + // just as wasteful. + + int b = buffer[ position++ ]; + + if( position >= bufferLength ) { + position = -1; + } // end if: end + + return b & 0xFF; // This is how you "cast" a byte that's + // intended to be unsigned. + } // end else + } // end if: position >= 0 + + // Else error + else { + throw new java.io.IOException( "Error in Base64 code reading stream." ); + } // end else + } // end read + + + /** + * Calls {@link #read()} repeatedly until the end of stream + * is reached or len bytes are read. + * Returns number of bytes read into array or -1 if + * end of stream is encountered. + * + * @param dest array to hold values + * @param off offset for array + * @param len max number of bytes to read into array + * @return bytes read into array or -1 if end of stream is encountered. + * @since 1.3 + */ + @Override + public int read( byte[] dest, int off, int len ) + throws java.io.IOException { + int i; + int b; + for( i = 0; i < len; i++ ) { + b = read(); + + if( b >= 0 ) { + dest[off + i] = (byte) b; + } + else if( i == 0 ) { + return -1; + } + else { + break; // Out of 'for' loop + } // Out of 'for' loop + } // end for: each byte read + return i; + } // end read + + } // end inner class InputStream + + + + + + + /* ******** I N N E R C L A S S O U T P U T S T R E A M ******** */ + + + + /** + * A {@link Base64.OutputStream} will write data to another + * java.io.OutputStream, given in the constructor, + * and encode/decode to/from Base64 notation on the fly. + * + * @see Base64 + * @since 1.3 + */ + public static class OutputStream extends java.io.FilterOutputStream { + + private boolean encode; + private int position; + private byte[] buffer; + private int bufferLength; + private int lineLength; + private boolean breakLines; + private byte[] b4; // Scratch used in a few places + private boolean suspendEncoding; + private int options; // Record for later + private byte[] decodabet; // Local copies to avoid extra method calls + + /** + * Constructs a {@link Base64.OutputStream} in ENCODE mode. + * + * @param out the java.io.OutputStream to which data will be written. + * @since 1.3 + */ + public OutputStream( java.io.OutputStream out ) { + this( out, ENCODE ); + } // end constructor + + + /** + * Constructs a {@link Base64.OutputStream} in + * either ENCODE or DECODE mode. + *

+ * Valid options:

+         *   ENCODE or DECODE: Encode or Decode as data is read.
+         *   DO_BREAK_LINES: don't break lines at 76 characters
+         *     (only meaningful when encoding)
+         *
+ *

+ * Example: new Base64.OutputStream( out, Base64.ENCODE ) + * + * @param out the java.io.OutputStream to which data will be written. + * @param options Specified options. + * @see Base64#ENCODE + * @see Base64#DECODE + * @see Base64#DO_BREAK_LINES + * @since 1.3 + */ + public OutputStream( java.io.OutputStream out, int options ) { + super( out ); + this.breakLines = (options & DO_BREAK_LINES) != 0; + this.encode = (options & ENCODE) != 0; + this.bufferLength = encode ? 3 : 4; + this.buffer = new byte[ bufferLength ]; + this.position = 0; + this.lineLength = 0; + this.suspendEncoding = false; + this.b4 = new byte[4]; + this.options = options; + this.decodabet = getDecodabet(options); + } // end constructor + + + /** + * Writes the byte to the output stream after + * converting to/from Base64 notation. + * When encoding, bytes are buffered three + * at a time before the output stream actually + * gets a write() call. + * When decoding, bytes are buffered four + * at a time. + * + * @param theByte the byte to write + * @since 1.3 + */ + @Override + public void write(int theByte) + throws java.io.IOException { + // Encoding suspended? + if( suspendEncoding ) { + this.out.write( theByte ); + return; + } // end if: supsended + + // Encode? + if( encode ) { + buffer[ position++ ] = (byte)theByte; + if( position >= bufferLength ) { // Enough to encode. + + this.out.write( encode3to4( b4, buffer, bufferLength, options ) ); + + lineLength += 4; + if( breakLines && lineLength >= MAX_LINE_LENGTH ) { + this.out.write( NEW_LINE ); + lineLength = 0; + } // end if: end of line + + position = 0; + } // end if: enough to output + } // end if: encoding + + // Else, Decoding + else { + // Meaningful Base64 character? + if( decodabet[ theByte & 0x7f ] > WHITE_SPACE_ENC ) { + buffer[ position++ ] = (byte)theByte; + if( position >= bufferLength ) { // Enough to output. + + int len = Base64.decode4to3( buffer, 0, b4, 0, options ); + out.write( b4, 0, len ); + position = 0; + } // end if: enough to output + } // end if: meaningful base64 character + else if( decodabet[ theByte & 0x7f ] != WHITE_SPACE_ENC ) { + throw new java.io.IOException( "Invalid character in Base64 data." ); + } // end else: not white space either + } // end else: decoding + } // end write + + + + /** + * Calls {@link #write(int)} repeatedly until len + * bytes are written. + * + * @param theBytes array from which to read bytes + * @param off offset for array + * @param len max number of bytes to read into array + * @since 1.3 + */ + @Override + public void write( byte[] theBytes, int off, int len ) + throws java.io.IOException { + // Encoding suspended? + if( suspendEncoding ) { + this.out.write( theBytes, off, len ); + return; + } // end if: supsended + + for( int i = 0; i < len; i++ ) { + write( theBytes[ off + i ] ); + } // end for: each byte written + + } // end write + + + + /** + * Method added by PHIL. [Thanks, PHIL. -Rob] + * This pads the buffer without closing the stream. + * @throws java.io.IOException if there's an error. + */ + public void flushBase64() throws java.io.IOException { + if( position > 0 ) { + if( encode ) { + out.write( encode3to4( b4, buffer, position, options ) ); + position = 0; + } // end if: encoding + else { + throw new java.io.IOException( "Base64 input not properly padded." ); + } // end else: decoding + } // end if: buffer partially full + + } // end flush + + + /** + * Flushes and closes (I think, in the superclass) the stream. + * + * @since 1.3 + */ + @Override + public void close() throws java.io.IOException { + // 1. Ensure that pending characters are written + flushBase64(); + + // 2. Actually close the stream + // Base class both flushes and closes. + super.close(); + + buffer = null; + out = null; + } // end close + + + + /** + * Suspends encoding of the stream. + * May be helpful if you need to embed a piece of + * base64-encoded data in a stream. + * + * @throws java.io.IOException if there's an error flushing + * @since 1.5.1 + */ + public void suspendEncoding() throws java.io.IOException { + flushBase64(); + this.suspendEncoding = true; + } // end suspendEncoding + + + /** + * Resumes encoding of the stream. + * May be helpful if you need to embed a piece of + * base64-encoded data in a stream. + * + * @since 1.5.1 + */ + public void resumeEncoding() { + this.suspendEncoding = false; + } // end resumeEncoding + + + + } // end inner class OutputStream + + +} // end class Base64 diff --git a/src/main/java/org/cobbzilla/util/string/HasLocale.java b/src/main/java/org/cobbzilla/util/string/HasLocale.java new file mode 100644 index 0000000..9df6704 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/string/HasLocale.java @@ -0,0 +1,9 @@ +package org.cobbzilla.util.string; + +import java.util.Locale; + +public interface HasLocale { + + Locale getLocale(); + +} diff --git a/src/main/java/org/cobbzilla/util/string/LocaleUtil.java b/src/main/java/org/cobbzilla/util/string/LocaleUtil.java new file mode 100644 index 0000000..4b298e9 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/string/LocaleUtil.java @@ -0,0 +1,130 @@ +package org.cobbzilla.util.string; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.io.StringReader; +import java.util.*; +import java.util.stream.Collectors; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.io.StreamUtil.stream2string; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.string.StringUtil.getPackagePath; + +@Slf4j +public class LocaleUtil { + + public static final String ISO_COUNTRY_CODES_RESOURCE = getPackagePath(LocaleUtil.class) + "/country_codes.txt"; + + public static final Set ISO_COUNTRY_CODES = Arrays.stream(stream2string(ISO_COUNTRY_CODES_RESOURCE).split("\n")) + .filter(cc -> !empty(cc) && !cc.trim().startsWith("#")) + .map(cc -> cc.trim().toLowerCase()) + .collect(Collectors.toSet()); + + public static boolean isValidCountryCode (String cc) { + return ISO_COUNTRY_CODES.contains(cc.toLowerCase()); + } + + public static final String TELEPHONE_CODES_RESOURCE = getPackagePath(LocaleUtil.class) + "/telephone_codes.txt"; + private static final Properties TELEPHONE_CODES_MAP = initPhoneCodes(); + private static Properties initPhoneCodes() { + try { + final Properties props = new Properties(); + props.load(new StringReader(stream2string(TELEPHONE_CODES_RESOURCE))); + return props; + } catch (Exception e) { + return die("initPhoneCodes: "+e, e); + } + } + + public static String getPhoneCode (String country) { + if (country == null) return null; + return TELEPHONE_CODES_MAP.getProperty(country.toUpperCase()); + } + + public static File findLocaleFile (File base, String locale) { + + if (empty(locale)) return base.exists() ? base : null; + + final String[] localeParts = locale.toLowerCase().replace("-", "_").split("_"); + final String lang = localeParts[0]; + final String region = localeParts.length > 1 ? localeParts[1] : null; + final String variant = localeParts.length > 2 ? localeParts[2] : null; + + File found; + if (!empty(variant)) { + found = findSpecificLocaleFile(base, lang + "_" + region + "_" + variant); + if (found != null) return found; + } + if (!empty(region)) { + found = findSpecificLocaleFile(base, lang + "_" + region); + if (found != null) return found; + } + found = findSpecificLocaleFile(base, lang); + if (found != null) return found; + + return base.exists() ? base : null; + } + + private static File findSpecificLocaleFile(File base, String locale) { + final String filename = base.getName(); + final int lastDot = filename.lastIndexOf('.'); + final String prefix; + final String suffix; + if (lastDot != -1) { + prefix = filename.substring(0, lastDot); + suffix = filename.substring(lastDot); + } else { + prefix = filename; + suffix = ""; + } + final File localeFile = new File(base.getParent(), prefix + "_" + locale + suffix); + return localeFile.exists() ? localeFile : null; + } + + public static Locale fromString(String localeString) { + final String[] parts = empty(localeString) ? StringUtil.EMPTY_ARRAY : localeString.split("[-_]+"); + switch (parts.length) { + case 3: return new Locale(parts[0], parts[1], parts[2]); + case 2: return new Locale(parts[0], parts[1]); + case 1: return new Locale(parts[0]); + case 0: return Locale.getDefault(); + default: + log.warn("fromString: invalid locale string: "+localeString); + return Locale.getDefault(); + } + } + + @Getter(lazy=true) private static final Map> defaultLocales = initDefaultLocales(); + private static Map> initDefaultLocales() { + final Map> defaults = new HashMap<>(); + final JsonNode node = json(stream2string(getPackagePath(LocaleUtil.class)+"/default_locales.json"), JsonNode.class); + return buildDefaultsMap(defaults, node); + } + public static List getDefaultLocales(String country) { return getDefaultLocales().get(country); } + + @Getter(lazy=true) private static final Map> defaultLanguages = initDefaultLanguages(); + private static Map> initDefaultLanguages() { + final Map> defaults = new HashMap<>(); + final JsonNode node = json(stream2string(getPackagePath(LocaleUtil.class)+"/default_langs.json"), JsonNode.class); + return buildDefaultsMap(defaults, node); + } + public static List getDefaultLanguages(String country) { return getDefaultLanguages().get(country); } + + private static Map> buildDefaultsMap(Map> defaults, JsonNode node) { + for (Iterator iter = node.fieldNames(); iter.hasNext(); ) { + final String country = iter.next(); + final JsonNode countryNode = node.get(country); + final List locales = new ArrayList<>(); + for (Iterator iter2 = countryNode.iterator(); iter2.hasNext(); ) { + locales.add(iter2.next().textValue()); + } + defaults.put(country, locales); + } + return defaults; + } +} diff --git a/src/main/java/org/cobbzilla/util/string/ResourceMessages.java b/src/main/java/org/cobbzilla/util/string/ResourceMessages.java new file mode 100644 index 0000000..4ba5e6f --- /dev/null +++ b/src/main/java/org/cobbzilla/util/string/ResourceMessages.java @@ -0,0 +1,30 @@ +package org.cobbzilla.util.string; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.util.ResourceBundle; + +@Slf4j +public abstract class ResourceMessages { + + protected String getBundleName() { return "labels/"+getClass().getSimpleName(); } + + @Getter(lazy=true) private final ResourceBundle bundle = ResourceBundle.getBundle(getBundleName()); + + // todo: add support for locale-specific bundles and messages + public String translate(String messageTemplate) { + + // strip leading/trailing curlies if they are there + while (messageTemplate.startsWith("{")) messageTemplate = messageTemplate.substring(1); + while (messageTemplate.endsWith("}")) messageTemplate = messageTemplate.substring(0, messageTemplate.length()-1); + + try { + return getBundle().getString(messageTemplate); + } catch (Exception e) { + log.warn("translate: Error looking up "+messageTemplate+": "+e); + return messageTemplate; + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/string/StringUtil.java b/src/main/java/org/cobbzilla/util/string/StringUtil.java new file mode 100644 index 0000000..a16f128 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/string/StringUtil.java @@ -0,0 +1,546 @@ +package org.cobbzilla.util.string; + +import com.google.common.base.CaseFormat; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.Transformer; +import org.apache.commons.io.input.ReaderInputStream; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.LocaleUtils; +import org.apache.commons.lang3.StringUtils; +import org.cobbzilla.util.javascript.JsEngine; +import org.cobbzilla.util.javascript.JsEngineConfig; +import org.cobbzilla.util.security.MD5Util; +import org.cobbzilla.util.time.JavaTimezone; +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; + +import java.io.StringReader; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.nio.charset.Charset; +import java.text.NumberFormat; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.apache.commons.lang.StringEscapeUtils.escapeSql; +import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; +import static org.cobbzilla.util.collection.ArrayUtil.arrayToString; +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.io.StreamUtil.loadResourceAsStringOrDie; + +public class StringUtil { + + public static final String UTF8 = "UTF-8"; + public static final Charset UTF8cs = Charset.forName(UTF8); + + public static final String EMPTY = ""; + public static final String[] EMPTY_ARRAY = {}; + public static final String DEFAULT_LOCALE = "en_US"; + public static final String BYTES_PATTERN = "(\\d+)(\\p{javaSpaceChar}+)?([MgGgTtPpEe][Bb])"; + public static final String CRLF = "\r\n"; + + public static final Transformer XFORM_TO_STRING = o -> String.valueOf(o); + + public static final String[] VOWELS = {"e", "a", "o", "i", "u"}; + public static boolean isVowel(String symbol) { return ArrayUtils.indexOf(VOWELS, symbol) != -1; } + + public static List toStringCollection (Collection c) { + return new ArrayList<>(CollectionUtils.collect(c, XFORM_TO_STRING)); + } + + public static String prefix(String s, int count) { + return s == null ? null : s.length() > count ? s.substring(0, count) : s; + } + + public static String packagePath(Class clazz) { + return clazz.getPackage().getName().replace(".","/"); + } + + public static String packagePath(String clazz) { return clazz.replace(".","/"); } + + public static List split (String s, String delim) { + final StringTokenizer st = new StringTokenizer(s, delim); + final List results = new ArrayList<>(); + while (st.hasMoreTokens()) { + results.add(st.nextToken()); + } + return results; + } + + public static String[] split2array (String s, String delim) { + final List vals = split(s, delim); + return vals.toArray(new String[vals.size()]); + } + + public static List splitLongs (String s, String delim) { + final StringTokenizer st = new StringTokenizer(s, delim); + final List results = new ArrayList<>(); + while (st.hasMoreTokens()) { + final String token = st.nextToken(); + results.add(empty(token) || token.equalsIgnoreCase("null") ? null : Long.parseLong(token)); + } + return results; + } + + public static List splitAndTrim (String s, String delim) { + final List results = new ArrayList<>(); + if (empty(s)) return results; + final StringTokenizer st = new StringTokenizer(s, delim); + while (st.hasMoreTokens()) { + results.add(st.nextToken().trim()); + } + return results; + } + + public static String replaceLast(String s, String find, String replace) { + if (empty(s)) return s; + int lastIndex = s.lastIndexOf(find); + if (lastIndex < 0) return s; + return s.substring(0, lastIndex) + s.substring(lastIndex).replaceFirst(find, replace); + } + + public static String lastPathElement(String url) { return url.substring(url.lastIndexOf("/")+1); } + + public static String safeShellArg (String s) { return s.replaceAll("[^-\\._ \t/=\\w]+", ""); } + public static String safeFunctionName (String s) { return s.replaceAll("\\W", ""); } + public static String safeSnakeName (String s) { return s.replaceAll("\\W", "_"); } + + public static String onlyDigits (String s) { return s.replaceAll("\\D+", ""); } + + public static String removeWhitespace (String s) { return s.replaceAll("\\p{javaSpaceChar}", ""); } + + public static Integer safeParseInt(String s) { + if (empty(s)) return null; + try { + return Integer.parseInt(s); + } catch (NumberFormatException e) { + return null; + } + } + + public static Double safeParseDouble(String s) { + if (empty(s)) return null; + try { + return Double.parseDouble(s); + } catch (NumberFormatException e) { + return null; + } + } + + public static String shortDateTime(String localeString, String timezone, long time) { + return formatDateTime("SS", localeString, timezone, time); + } + + public static String mediumDateTime(String localeString, String timezone, long time) { + return formatDateTime("MM", localeString, timezone, time); + } + + public static String fullDateTime(String localeString, String timezone, long time) { + return formatDateTime("FF", localeString, timezone, time); + } + + public static String formatDateTime(String style, String localeString, String timezone, long time) { + final Locale locale = LocaleUtils.toLocale(localeString); + final JavaTimezone tz = JavaTimezone.fromString(timezone); + return DateTimeFormat.forPattern(DateTimeFormat.patternForStyle(style, locale)) + .withZone(DateTimeZone.forTimeZone(tz.getTimeZone())).print(time); + } + + public static final long HOUR = TimeUnit.HOURS.toMillis(1); + public static final long MINUTE = TimeUnit.MINUTES.toMillis(1); + public static final long SECOND = TimeUnit.SECONDS.toMillis(1); + + public static String chopSuffix(String val) { return val.substring(0, val.length()-1); } + + public static String chopToFirst(String val, String find) { + return !val.contains(find) ? val : val.substring(val.indexOf(find) + find.length()); + } + + public static String trimQuotes (String s) { + if (s == null) return s; + while (s.startsWith("\"") || s.startsWith("\'")) s = s.substring(1); + while (s.endsWith("\"") || s.endsWith("\'")) s = s.substring(0, s.length()-1); + return s; + } + + public static boolean endsWithAny(String s, String[] suffixes) { + if (s == null) return false; + for (String suffix : suffixes) if (s.endsWith(suffix)) return true; + return false; + } + + public static String getPackagePath(Class clazz) { + return clazz.getPackage().getName().replace('.', '/'); + } + + public static String repeat (String s, int n) { + return new String(new char[n*s.length()]).replace("\0", s); + } + + public static String urlEncode (String s) { + try { + return URLEncoder.encode(s, UTF8); + } catch (UnsupportedEncodingException e) { + return die("urlEncode: "+e, e); + } + } + + public static String simpleUrlEncode (String s) { + return s.replace("&", "&").replace("<", "<").replace(">", ">"); + } + + public static String urlDecode (String s) { + try { + return URLDecoder.decode(s, UTF8); + } catch (UnsupportedEncodingException e) { + return die("urlDecode: "+e, e); + } + } + + public static URI uriOrDie (String s) { + try { + return new URI(s); + } catch (URISyntaxException e) { + return die("bad uri: "+e, e); + } + } + + public static String urlParameterize(Map params) { + final StringBuilder sb = new StringBuilder(); + for (Map.Entry entry : params.entrySet()) { + if (sb.length() > 0) sb.append('&'); + sb.append(urlEncode(entry.getKey())) + .append('=') + .append(urlEncode(entry.getValue())); + } + return sb.toString(); + } + + public static String toString (Collection c) { return toString(c, ","); } + + public static String toString (Collection c, String sep) { + return toString(c, sep, null); + } + + public static String toString (Collection c, String sep, Function transformer) { + StringBuilder builder = new StringBuilder(); + for (Object o : c) { + if (builder.length() > 0) builder.append(sep); + builder.append(transformer != null ? transformer.apply(o) : o); + } + return builder.toString(); + } + + public static String sqlIn (Collection c) { return toString(c, ",", ESCAPE_SQL); } + + public static final Function ESCAPE_SQL = o -> "'"+o.toString().replace("\'", "\'\'")+"'"; + + public static String toString(Map map) { + if (map == null) return "null"; + final StringBuilder b = new StringBuilder("{"); + for (Object key : map.keySet()) { + final Object value = map.get(key); + b.append(key).append("="); + if (value == null) { + b.append("null"); + } else { + if (value.getClass().isArray()) { + b.append(arrayToString((Object[]) value, ", ")); + } else if (value instanceof Map) { + b.append(toString((Map) value)); + } else if (value instanceof Collection) { + b.append(toString((Collection) value, ", ")); + } else { + b.append(value); + } + } + } + return b.append("}").toString(); + } + + public static Set toSet (String s, String sep) { + return new HashSet<>(Arrays.asList(s.split(sep))); + } + + public static String tohex(byte[] data) { + return tohex(data, 0, data.length); + } + + public static String tohex(byte[] data, int start, int len) { + StringBuilder b = new StringBuilder(); + int stop = start+len; + for (int i=start; i> 4) + 16) % 16] + MD5Util.HEX_DIGITS[(i + 128) % 16]; + } + + public static String uncapitalize(String s) { + return empty(s) ? s : s.length() == 1 ? s.toLowerCase() : s.substring(0, 1).toLowerCase() + s.substring(1); + } + + public static String pluralize(String val) { + if (empty(val)) return val; + if (val.endsWith("y")) { + return val.substring(0, val.length()-1)+"ies"; + } else if (!val.endsWith("s")) { + return val + "s"; + } + return val; + } + + public static boolean exceptionContainsMessage(Throwable e, String s) { + return e != null && ( + (e.getMessage() != null && e.getMessage().contains(s)) + || (e.getCause() != null && exceptionContainsMessage(e.getCause(), s)) + ); + } + + public static String ellipsis(String s, int len) { + if (s == null || s.length() <= len) return s; + return s.substring(0, len-3) + "..."; + } + + public static String truncate(String s, int len) { + if (s == null || s.length() <= len) return s; + return s.substring(0, len); + } + + public static boolean containsIgnoreCase(Collection values, String value) { + for (String v : values) if (v != null && v.equalsIgnoreCase(value)) return true; + return false; + } + + /** + * Return what the default "property name" would be for this thing, if named according to its type + * @param thing the thing to look at + * @param the type of thing it is + * @return the class name of the thing with the first letter downcased + */ + public static String classAsFieldName(T thing) { + return uncapitalize(thing.getClass().getSimpleName()); + } + + /** + * Split a string into multiple query terms, respecting quotation marks + * @param query The query string + * @return a List of query terms + */ + public static List splitIntoTerms(String query) { + final List terms = new ArrayList<>(); + final StringTokenizer st = new StringTokenizer(query, "\n\t \"", true); + + StringBuilder current = new StringBuilder(); + boolean inQuotes = false; + while (st.hasMoreTokens()) { + final String token = st.nextToken(); + if (token.equals("\"")) { + String term = current.toString().trim(); + if (term.length() > 0) terms.add(term); + current = new StringBuilder(); + inQuotes = !inQuotes; + + } else if (token.matches("\\s+")) { + if (inQuotes && !current.toString().endsWith(" ")) current.append(" "); + + } else { + if (inQuotes) { + current.append(token); + } else { + terms.add(token); + } + } + } + if (current.length() > 0) terms.add(current.toString().trim()); + return terms; + } + + public static String chop(String input, String chopIfSuffix) { + return input.endsWith(chopIfSuffix) ? input.substring(0, input.length()-chopIfSuffix.length()) : input; + } + + public static boolean isNumber(String val) { + if (val == null) return false; + val = val.trim(); + try { + Double.parseDouble(val); + return true; + } catch (Exception ignored) {} + try { + Long.parseLong(val); + return true; + } catch (Exception ignored) {} + return false; + } + + public static boolean isPunctuation(char c) { + return c == '.' || c == ',' || c == '?' || c == '!' || c == ';' || c == ':'; + } + + public static boolean hasScripting(String value) { + if (empty(value)) return false; + value = value.toLowerCase().replace("<", "<"); + final String nospace = removeWhitespace(value); + return nospace.contains(" 10) ? "."+ (cents % 100) : ".0"+(cents % 100)); } + + public static String hexPath(String hex, int count) { + final StringBuilder b = new StringBuilder(); + for (int i=0; i 0) b.append("/"); + b.append(hex.substring(i*2, i*2 + 2)); + } + return b.toString(); + } + + public static final NumberFormat NUMBER_FORMAT = NumberFormat.getInstance(); + public static String formatDollars(long value, boolean sign) { return empty(value) ? "" : (sign?"$":"") + NUMBER_FORMAT.format(value); } + + public static String formatDollars(long value) { return formatDollars(value, true); } + public static String formatDollarsWithSign(long value) { return formatDollars(value, true); } + public static String formatDollarsNoSign(long value) { return formatDollars(value, false); } + + public static String formatDollarsAndCents(long value) { return formatDollarsAndCents(value, true); } + public static String formatDollarsAndCentsWithSign(long value) { return formatDollarsAndCents(value, true); } + public static String formatDollarsAndCentsNoSign(long value) { return formatDollarsAndCents(value, false); } + + public static String formatDollarsAndCents(long value, boolean sign) { + return empty(value) ? "" : (sign?"$":"") + NUMBER_FORMAT.format(value/100) + + (value % 100 == 0 ? ".00" : "."+(value%100<10 ? "0"+(value%100) : (value%100))); + } + + public static String formatDollarsAndCentsPlain(long value) { + return "" + (value/100) + (value % 100 == 0 ? ".00" : "."+(value%100<10 ? "0"+(value%100) : (value%100))); + } + + public static int parseToCents(String amount) { + if (empty(amount)) return die("getDownAmountCents: downAmount was empty"); + String val = amount.trim(); + int dotPos = val.indexOf("."); + if (dotPos == val.length()) { + val = val.substring(0, val.length()-1); + dotPos = -1; + } + if (dotPos == -1) return 100 * Integer.parseInt(val); + return (100 * Integer.parseInt(val.substring(0, dotPos))) + Integer.parseInt(val.substring(dotPos+1)); + } + + public static double parsePercent (String pct) { + if (empty(pct)) die("parsePercent: "+pct); + return Double.parseDouble(chop(removeWhitespace(pct), "%")); + } + + public static ReaderInputStream stream(String data) { + return new ReaderInputStream(new StringReader(data), UTF8cs); + } + + public static String firstMatch(String s, String regex) { + final Pattern p = Pattern.compile(regex); + final Matcher m = p.matcher(s); + return m.find() ? m.group(0) : null; + } + + private static final String DIFF_JS + = loadResourceAsStringOrDie(getPackagePath(StringUtil.class)+"/diff_match_patch.js") + "\n" + + loadResourceAsStringOrDie(getPackagePath(StringUtil.class)+"/calc_diff.js") + "\n"; + public static JsEngine DIFF_JS_ENGINE = new JsEngine(new JsEngineConfig(5, 20, null)); + public static String diff (String text1, String text2, Map opts) { + if (opts == null) opts = new HashMap<>(); + final Map ctx = new HashMap<>(); + ctx.put("text1", text1); + ctx.put("text2", text2); + ctx.put("opts", opts); + return DIFF_JS_ENGINE.evaluate(DIFF_JS, ctx); + } + + public static String replaceWithRandom(String s, String find, int randLength) { + while (s.contains(find)) s = s.replaceFirst(find, randomAlphanumeric(randLength)); + return s; + } + + public static String firstWord(String value) { return value.trim().split("\\p{javaSpaceChar}+")[0]; } + + /** + * If both strings are empty (null or empty string) return true, else use apache's StringUtils.equals method. + */ + public static boolean equalsExtended(String s1, String s2) { + return (empty(s1) && empty(s2)) || StringUtils.equals(s1, s2); + } + + public static final String PCT = "%"; + public static final String ESC_PCT = "[%]"; + public static String sqlFilter(String value) { + // escape any embedded '%' chars, and then add '%' as the first and last chars + // also replace any embedded single-quote characters with '%', this helps prevent SQL injection attacks + return PCT + value.toLowerCase().replace(PCT, ESC_PCT).replace("'", PCT) + PCT; + } + + public static String sqlEscapeAndQuote(String val) { return "'" + escapeSql(val) + "'"; } + +} diff --git a/src/main/java/org/cobbzilla/util/string/ValidationRegexes.java b/src/main/java/org/cobbzilla/util/string/ValidationRegexes.java new file mode 100644 index 0000000..e11b3e9 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/string/ValidationRegexes.java @@ -0,0 +1,82 @@ +package org.cobbzilla.util.string; + +import org.cobbzilla.util.collection.MapBuilder; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.cobbzilla.util.string.StringUtil.chop; + +public class ValidationRegexes { + + public static final Pattern LOGIN_PATTERN = pattern("^[\\w\\-]+$"); + public static final Pattern EMAIL_PATTERN = pattern("^[A-Z0-9][A-Z0-9._%+-]*@[A-Z0-9.-]+\\.[A-Z]{2,6}$"); + public static final Pattern EMAIL_NAME_PATTERN = pattern("^[A-Z0-9][A-Z0-9._%+-]*$"); + + public static final Pattern[] LOCALE_PATTERNS = { + pattern("^[a-zA-Z]{2,3}([-_][a-zA-z]{2}(@[\\w]+)?)?"), // ubuntu style: en_US or just en + pattern("^[a-zA-Z]{2,3}([-_][\\w]+)?"), // some apps use style: ca-valencia + }; + + public static final String UUID_REGEX = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; + public static final Pattern UUID_PATTERN = pattern(UUID_REGEX); + + public static final String NUMERIC_REGEX = "\\d+"; + public static final Pattern NUMERIC_PATTERN = pattern(NUMERIC_REGEX); + + public static final int IP4_MAXLEN = 15; + public static final int IP6_MAXLEN = 45; + + public static final Pattern IPv4_PATTERN = pattern("^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}$"); + public static final Pattern IPv6_PATTERN = pattern("^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))$"); + + public static final String HOST_REGEX = "^([A-Z0-9]{1,63}|[A-Z0-9][A-Z0-9\\-]{0,61}[A-Z0-9])(\\.([A-Z0-9]{1,63}|[A-Z0-9][A-Z0-9\\-]{0,61}[A-Z0-9]))*$"; + public static final Pattern HOST_PATTERN = pattern(HOST_REGEX); + public static final Pattern HOST_PART_PATTERN = pattern("^([A-Z0-9]|[A-Z0-9][A-Z0-9\\-]{0,61}[A-Z0-9])$"); + public static final Pattern PORT_PATTERN = pattern("^[\\d]{1,5}$"); + + public static final String DOMAIN_REGEX = "^([A-Z0-9]{1,63}|[A-Z0-9][A-Z0-9\\-]{0,61}[A-Z0-9])(\\.([A-Z0-9]{1,63}|[A-Z0-9][A-Z0-9\\-]{0,61}[A-Z0-9]))+$"; + public static final Pattern DOMAIN_PATTERN = pattern(DOMAIN_REGEX); + + public static final String URL_REGEX = "(https?|ftp|file)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]"; + public static final String URL_REGEX_ONLY = "^" + URL_REGEX + "$"; + public static final Pattern URL_PATTERN = pattern(URL_REGEX_ONLY); + public static final Pattern HTTP_PATTERN = pattern("^(https?)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]$"); + public static final Pattern HTTPS_PATTERN = pattern("^https://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]$"); + + public static final String VARNAME_REGEX = "^[A-Za-z][_A-Za-z0-9]+$"; + public static final Pattern VARNAME_PATTERN = pattern(VARNAME_REGEX); + + public static final Pattern FILENAME_PATTERN = pattern("^[_A-Z0-9\\-\\.]+$"); + public static final Pattern INTEGER_PATTERN = pattern("^[0-9]+$"); + public static final Pattern DECIMAL_PATTERN = pattern("^[0-9]+(\\.[0-9]+)?$"); + + public static final Map PHONE_PATTERNS = MapBuilder.build(new Object[][]{ + {"US", pattern("^\\d{10}$")} + }); + public static final Pattern DEFAULT_PHONE_PATTERN = pattern("^\\d+([-\\.\\s]?\\d+?){8,}[\\d]+$"); + + public static final String YYYYMMDD_REGEX = "^(19|20|21)[0-9]{2}-[01][0-9]-(0[1-9]|[1-2][0-9]|3[0-1])$"; + public static final Pattern YYYYMMDD_PATTERN = pattern(YYYYMMDD_REGEX); + + public static final String ZIPCODE_REGEX = "^\\d{5}(-\\d{4})?$"; + public static final Pattern ZIPCODE_PATTERN = pattern(ZIPCODE_REGEX); + + public static Pattern pattern(String regex) { return Pattern.compile(regex, Pattern.CASE_INSENSITIVE); } + + public static List findAllRegexMatches(String text, String regex) { + if (regex.startsWith("^")) regex = regex.substring(1); + if (regex.endsWith("$")) regex = chop(regex, "$"); + regex = "(.*(?"+ regex +")+.*)+"; + final List found = new ArrayList<>(); + final Matcher matcher = Pattern.compile(regex, Pattern.MULTILINE).matcher(text); + while (matcher.find()) { + found.add(matcher.group("match")); + } + return found; + } + +} diff --git a/src/main/java/org/cobbzilla/util/system/Bytes.java b/src/main/java/org/cobbzilla/util/system/Bytes.java new file mode 100644 index 0000000..0b8c1f2 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/system/Bytes.java @@ -0,0 +1,63 @@ +package org.cobbzilla.util.system; + +import java.text.DecimalFormat; + +import static org.apache.commons.lang3.StringUtils.chop; +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.string.StringUtil.removeWhitespace; + +public class Bytes { + + public static final long KB = 1024; + public static final long MB = 1024 * KB; + public static final long GB = 1024 * MB; + public static final long TB = 1024 * GB; + public static final long PB = 1024 * TB; + public static final long EB = 1024 * PB; + + public static final long KiB = 1000; + public static final long MiB = 1000 * KiB; + public static final long GiB = 1000 * MiB; + public static final long TiB = 1000 * GiB; + public static final long PiB = 1000 * TiB; + public static final long EiB = 1000 * PiB; + + public static long parse(String value) { + String val = removeWhitespace(value).toLowerCase(); + if (val.endsWith("bytes")) return Long.parseLong(val.substring(0, val.length()-"bytes".length())); + if (val.endsWith("b")) return Long.parseLong(chop(val)); + final char suffix = val.charAt(val.length()); + final long size = Long.parseLong(val.substring(0, val.length() - 1)); + switch (suffix) { + case 'k': return KB * size; + case 'm': return MB * size; + case 'g': return GB * size; + case 't': return TB * size; + case 'p': return PB * size; + case 'e': return EB * size; + default: return die("parse: Unrecognized suffix '"+suffix+"' in string "+value); + } + } + + public static final DecimalFormat DEFAULT_FORMAT = new DecimalFormat(); + static { + DEFAULT_FORMAT.setMaximumFractionDigits(2); + } + + public static String format(Long count) { + if (count == null) return "0 bytes"; + if (count >= EB) return DEFAULT_FORMAT.format(count.doubleValue() / ((double) EB)) + " EB"; + if (count >= PB) return DEFAULT_FORMAT.format(count.doubleValue() / ((double) PB)) + " PB"; + if (count >= TB) return DEFAULT_FORMAT.format(count.doubleValue() / ((double) TB)) + " TB"; + if (count >= GB) return DEFAULT_FORMAT.format(count.doubleValue() / ((double) GB)) + " GB"; + if (count >= MB) return DEFAULT_FORMAT.format(count.doubleValue() / ((double) MB)) + " MB"; + if (count >= KB) return DEFAULT_FORMAT.format(count.doubleValue() / ((double) KB)) + " KB"; + return count + " bytes"; + } + + public static String formatBrief(Long count) { + final String s = format(count); + return s.endsWith(" bytes") ? s.split("\\w+")[0]+"b" : removeWhitespace(s.toLowerCase()); + } + +} diff --git a/src/main/java/org/cobbzilla/util/system/Command.java b/src/main/java/org/cobbzilla/util/system/Command.java new file mode 100644 index 0000000..5747346 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/system/Command.java @@ -0,0 +1,72 @@ +package org.cobbzilla.util.system; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; +import org.apache.commons.exec.CommandLine; +import org.apache.commons.lang3.ArrayUtils; +import org.cobbzilla.util.collection.SingletonList; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.string.StringUtil.UTF8cs; + +@NoArgsConstructor @Accessors(chain=true) +public class Command { + + public static final List DEFAULT_EXIT_VALUES = new SingletonList<>(0); + public static final int[] DEFAULT_EXIT_VALUES_INT = { 0 }; + + @Getter @Setter private CommandLine commandLine; + @Getter @Setter private String input; + @Getter @Setter private byte[] rawInput; + @Getter @Setter private InputStream stdin; + @Getter @Setter private File dir; + @Getter @Setter private Map env; + private List exitValues = DEFAULT_EXIT_VALUES; + + @Getter @Setter private boolean copyToStandard = false; + + @Getter @Setter private OutputStream out; + public boolean hasOut () { return out != null; } + + @Getter @Setter private OutputStream err; + public boolean hasErr () { return err != null; } + + public Command(CommandLine commandLine) { this.commandLine = commandLine; } + public Command(String command) { this(CommandLine.parse(command)); } + public Command(File executable) { this(abs(executable)); } + + public boolean hasDir () { return dir != null; } + public boolean hasInput () { return !empty(input) || !empty(rawInput) || stdin != null; } + public InputStream getInputStream () { + if (!hasInput()) return null; + if (stdin != null) return stdin; + if (rawInput != null) return new ByteArrayInputStream(rawInput); + return new ByteArrayInputStream(input.getBytes(UTF8cs)); + } + + public int[] getExitValues () { + return exitValues == DEFAULT_EXIT_VALUES + ? DEFAULT_EXIT_VALUES_INT + : ArrayUtils.toPrimitive(exitValues.toArray(new Integer[exitValues.size()])); + } + + public Command setExitValues (List values) { this.exitValues = values; return this; } + + public Command setExitValues (int[] values) { + exitValues = new ArrayList<>(values.length); + for (int v : values) exitValues.add(v); + return this; + } + +} diff --git a/src/main/java/org/cobbzilla/util/system/CommandProgressCallback.java b/src/main/java/org/cobbzilla/util/system/CommandProgressCallback.java new file mode 100644 index 0000000..c85bebf --- /dev/null +++ b/src/main/java/org/cobbzilla/util/system/CommandProgressCallback.java @@ -0,0 +1,11 @@ +package org.cobbzilla.util.system; + +public interface CommandProgressCallback { + + /** + * Called when a CommandProgressFilter encounters a new indicator + * @param marker Contains the percent done, pattern that matched, and the specific line that matched + */ + public void updateProgress(CommandProgressMarker marker); + +} diff --git a/src/main/java/org/cobbzilla/util/system/CommandProgressFilter.java b/src/main/java/org/cobbzilla/util/system/CommandProgressFilter.java new file mode 100644 index 0000000..6e98f62 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/system/CommandProgressFilter.java @@ -0,0 +1,49 @@ +package org.cobbzilla.util.system; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; +import org.apache.tools.ant.util.LineOrientedOutputStream; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +@Accessors(chain=true) +public class CommandProgressFilter extends LineOrientedOutputStream { + + @Getter private int pctDone = 0; + @Getter private int indicatorPos = 0; + @Getter private boolean closed = false; + @Getter @Setter private CommandProgressCallback callback; + + @AllArgsConstructor + private class CommandProgressIndicator { + @Getter @Setter private int percent; + @Getter @Setter private Pattern pattern; + } + + private List indicators = new ArrayList<>(); + + public CommandProgressFilter addIndicator(String pattern, int pct) { + indicators.add(new CommandProgressIndicator(pct, Pattern.compile(pattern))); + return this; + } + + @Override public void close() throws IOException { closed = true; } + + @Override protected void processLine(String line) throws IOException { + for (int i=indicatorPos; i loadShellExports (String path) throws IOException { + if (!path.startsWith("/")) { + final File file = userFile(path); + if (file.exists()) return loadShellExports(file); + } + return loadShellExports(new File(path)); + } + + public static File userFile(String path) { + return new File(System.getProperty("user.home") + File.separator + path); + } + + public static Map loadShellExports (File f) throws IOException { + try (InputStream in = new FileInputStream(f)) { + return loadShellExports(in); + } + } + + public static Map loadShellExports (InputStream in) throws IOException { + final Map map = new HashMap<>(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { + String line, key, value; + int eqPos; + while ((line = reader.readLine()) != null) { + line = line.trim(); + if (line.startsWith("#")) continue; + if (line.startsWith(EXPORT_PREFIX)) { + line = line.substring(EXPORT_PREFIX.length()).trim(); + eqPos = line.indexOf('='); + if (eqPos != -1) { + key = line.substring(0, eqPos).trim(); + value = line.substring(eqPos+1).trim(); + value = trimQuotes(value); + map.put(key, value); + } + } + } + } + return map; + } + + public static Map loadShellExportsOrDie (String f) { + try { return loadShellExports(f); } catch (Exception e) { + return die("loadShellExportsOrDie: "+e); + } + } + + public static Map loadShellExportsOrDie (File f) { + try { return loadShellExports(f); } catch (Exception e) { + return die("loadShellExportsOrDie: "+e); + } + } + + public static void replaceShellExport (String f, String name, String value) throws IOException { + replaceShellExports(new File(f), MapBuilder.build(name, value)); + } + + public static void replaceShellExport (File f, String name, String value) throws IOException { + replaceShellExports(f, MapBuilder.build(name, value)); + } + + public static void replaceShellExports (String f, Map exports) throws IOException { + replaceShellExports(new File(f), exports); + } + + public static void replaceShellExports (File f, Map exports) throws IOException { + + // validate -- no quote chars allowed for security reasons + for (String key : exports.keySet()) { + if (key.contains("\"") || key.contains("\'")) throw new IllegalArgumentException("replaceShellExports: name cannot contain a quote character: "+key); + String value = exports.get(key); + if (value.contains("\"") || value.contains("\'")) throw new IllegalArgumentException("replaceShellExports: value for "+key+" cannot contain a quote character: "+value); + } + + // read entire file as a string + final String contents = FileUtil.toString(f); + + // walk file line by line and look for replacements to make, overwrite file. + final Set replaced = new HashSet<>(exports.size()); + try (Writer w = new FileWriter(f)) { + for (String line : contents.split("\n")) { + line = line.trim(); + boolean found = false; + for (String key : exports.keySet()) { + if (!line.startsWith("#") && line.matches("^\\s*export\\s+" + key + "\\s*=.*")) { + w.write("export " + key + "=\"" + exports.get(key) + "\""); + replaced.add(key); + found = true; + break; + } + } + if (!found) w.write(line); + w.write("\n"); + } + + for (String key : exports.keySet()) { + if (!replaced.contains(key)) { + w.write("export "+key+"=\""+exports.get(key)+"\"\n"); + } + } + } + } + + public static MultiCommandResult exec (Collection commands) throws IOException { + final MultiCommandResult result = new MultiCommandResult(); + for (String c : commands) { + Command command = new Command(c); + result.add(command, exec(c)); + if (result.hasException()) return result; + } + return result; + } + + public static CommandResult exec (String command) throws IOException { + return exec(CommandLine.parse(command)); + } + + public static CommandResult exec (CommandLine command) throws IOException { + return exec(new Command(command)); + } + + public static CommandResult exec (Command command) throws IOException { + + final DefaultExecutor executor = new DefaultExecutor(); + + final ByteArrayOutputStream outBuffer = new ByteArrayOutputStream(); + OutputStream out = command.hasOut() ? new TeeOutputStream(outBuffer, command.getOut()) : outBuffer; + if (command.isCopyToStandard()) out = new TeeOutputStream(out, System.out); + + final ByteArrayOutputStream errBuffer = new ByteArrayOutputStream(); + OutputStream err = command.hasErr() ? new TeeOutputStream(errBuffer, command.getErr()) : errBuffer; + if (command.isCopyToStandard()) err = new TeeOutputStream(err, System.err); + + final ExecuteStreamHandler handler = new PumpStreamHandler(out, err, command.getInputStream()); + executor.setStreamHandler(handler); + + if (command.hasDir()) executor.setWorkingDirectory(command.getDir()); + executor.setExitValues(command.getExitValues()); + + try { + final int exitValue = executor.execute(command.getCommandLine(), command.getEnv()); + return new CommandResult(exitValue, outBuffer, errBuffer); + + } catch (Exception e) { + final String stdout = outBuffer.toString().trim(); + final String stderr = errBuffer.toString().trim(); + log.error("exec("+command.getCommandLine()+"): " + e + + (stdout.length() > 0 ? "\nstdout="+ellipsis(stdout, 1000) : "") + + (stderr.length() > 0 ? "\nstderr="+ellipsis(stderr, 1000) : "")); + return new CommandResult(e, outBuffer, errBuffer); + } + } + + public static int chmod (File file, String perms) { + return chmod(abs(file), perms, false); + } + + public static int chmod (File file, String perms, boolean recursive) { + return chmod(abs(file), perms, recursive); + } + + public static int chmod (String file, String perms) { + return chmod(file, perms, false); + } + + public static int chmod (String file, String perms, boolean recursive) { + final CommandLine commandLine = new CommandLine(CHMOD); + if (recursive) commandLine.addArgument("-R"); + commandLine.addArgument(perms); + commandLine.addArgument(abs(file), false); + final Executor executor = new DefaultExecutor(); + try { + return executor.execute(commandLine); + } catch (Exception e) { + return die("chmod: "+e, e); + } + } + + public static int chgrp(String group, File path) { + return chgrp(group, path, false); + } + + public static int chgrp(String group, File path, boolean recursive) { + return chgrp(group, abs(path), recursive); + } + + public static int chgrp(String group, String path) { + return chgrp(group, path, false); + } + + public static int chgrp(String group, String path, boolean recursive) { + return runChCmd(group, path, recursive, CHGRP); + } + + private static int runChCmd(String subject, String path, boolean recursive, String cmd) { + final Executor executor = new DefaultExecutor(); + final CommandLine command = new CommandLine(cmd); + if (recursive) command.addArgument("-R"); + command.addArgument(subject).addArgument(path); + try { + return executor.execute(command); + } catch (Exception e) { + return die(cmd+": "+e, e); + } + } + + public static int chown(String owner, File path) { return chown(owner, path, false); } + + public static int chown(String owner, File path, boolean recursive) { + return chown(owner, abs(path), recursive); + } + + public static int chown(String owner, String path) { return chown(owner, path, false); } + + public static int chown(String owner, String path, boolean recursive) { + return runChCmd(owner, path, recursive, CHOWN); + } + + public static String toString(String command) { + try { + return exec(command).getStdout().trim(); + } catch (IOException e) { + return die("Error executing: "+command+": "+e, e); + } + } + + public static String hostname () { return toString("hostname"); } + public static String domainname() { return toString("hostname -d"); } + public static String hostname_short() { return toString("hostname -s"); } + public static String whoami() { return toString("whoami"); } + public static boolean isRoot() { return "root".equals(whoami()); } + + public static String locale () { + return execScript("locale | grep LANG= | tr '=.' ' ' | awk '{print $2}'").trim(); + } + + public static String lang () { + return execScript("locale | grep LANG= | tr '=_' ' ' | awk '{print $2}'").trim(); + } + + public static File tempScript (String contents) { + contents = "#!/bin/bash\n\n"+contents; + try { + final File temp = File.createTempFile("tempScript", ".sh", getDefaultTempDir()); + FileUtil.toFile(temp, contents); + chmod(temp, "700"); + return temp; + + } catch (Exception e) { + return die("tempScript("+contents+") failed: "+e, e); + } + } + + public static String execScript (String contents) { return execScript(contents, null); } + + public static String execScript (String contents, Map env) { return execScript(contents, env, null); } + + public static String execScript (String contents, Map env, List exitValues) { + final CommandResult result = scriptResult(contents, env, null, exitValues); + if (!result.isZeroExitStatus() && (exitValues == null || !exitValues.contains(result.getExitStatus()))) { + die("execScript: non-zero exit: "+result); + } + return result.getStdout(); + } + + public static CommandResult scriptResult (String contents) { return scriptResult(contents, null, null, null); } + + public static CommandResult scriptResult (String contents, Map env) { + return scriptResult(contents, env, null, null); + } + + public static CommandResult scriptResult (String contents, String input) { + return scriptResult(contents, null, input, null); + } + + public static CommandResult scriptResult (String contents, Map env, String input, List exitValues) { + try { + @Cleanup("delete") final File script = tempScript(contents); + final Command command = new Command(new CommandLine(script)).setEnv(env).setInput(input); + if (!empty(exitValues)) command.setExitValues(exitValues); + return exec(command); + } catch (Exception e) { + return die("Error executing: "+e); + } + } + + public static CommandResult okResult(CommandResult result) { + if (result == null || !result.isZeroExitStatus()) die("error: "+result); + return result; + } + + public static File home(String user) { + String path = execScript("cd ~" + user + " && pwd"); + if (empty(path)) die("home("+user+"): no home found for user "+user); + final File f = new File(path); + if (!f.exists()) die("home("+user+"): home does not exist "+path); + return f; + } + + public static File pwd () { return new File(System.getProperty("user.dir")); } + +} diff --git a/src/main/java/org/cobbzilla/util/system/ConnectionInfo.java b/src/main/java/org/cobbzilla/util/system/ConnectionInfo.java new file mode 100644 index 0000000..6370c2a --- /dev/null +++ b/src/main/java/org/cobbzilla/util/system/ConnectionInfo.java @@ -0,0 +1,25 @@ +package org.cobbzilla.util.system; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@NoArgsConstructor @AllArgsConstructor +public class ConnectionInfo { + + @Getter @Setter private String host; + @Getter @Setter private Integer port; + public boolean hasPort () { return port != null; } + + @Getter @Setter private String username; + @Getter @Setter private String password; + + // convenience methods + public String getUser () { return getUsername(); } + public void setUser (String user) { setUsername(user); } + + public ConnectionInfo (String host, Integer port) { + this(host, port, null, null); + } +} diff --git a/src/main/java/org/cobbzilla/util/system/MultiCommandResult.java b/src/main/java/org/cobbzilla/util/system/MultiCommandResult.java new file mode 100644 index 0000000..6c6bcd0 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/system/MultiCommandResult.java @@ -0,0 +1,27 @@ +package org.cobbzilla.util.system; + +import lombok.Getter; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class MultiCommandResult { + + @Getter private final Map results = new LinkedHashMap<>(); + + public boolean hasException () { + for (CommandResult result : results.values()) { + if (result.hasException()) return true; + } + return false; + } + + public void add(Command command, CommandResult commandResult) { results.put(command, commandResult); } + + @Override public String toString() { + return "MultiCommandResult{" + + "results=" + results + + ", exception=" + hasException() + + '}'; + } +} diff --git a/src/main/java/org/cobbzilla/util/system/OsType.java b/src/main/java/org/cobbzilla/util/system/OsType.java new file mode 100644 index 0000000..0dbd0fa --- /dev/null +++ b/src/main/java/org/cobbzilla/util/system/OsType.java @@ -0,0 +1,43 @@ +package org.cobbzilla.util.system; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.sun.jna.Platform; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; + +public enum OsType { + + windows, macosx, linux; + + @JsonCreator public static OsType fromString (String val) { return valueOf(val.toLowerCase()); } + + public static final OsType CURRENT_OS = initCurrentOs(); + + private static OsType initCurrentOs() { + if (Platform.isWindows()) return windows; + if (Platform.isMac()) return macosx; + if (Platform.isLinux()) return linux; + return die("could not determine operating system: "+System.getProperty("os.name")); + } + + public static final boolean IS_ADMIN = initIsAdmin(); + + private static boolean initIsAdmin() { + switch (CURRENT_OS) { + case macosx: case linux: return System.getProperty("user.name").equals("root"); + case windows: return WindowsAdminUtil.isUserWindowsAdmin(); + default: return false; + } + } + + public static final String ADMIN_USERNAME = initAdminUsername(); + + private static String initAdminUsername() { + switch (CURRENT_OS) { + case macosx: case linux: return "root"; + case windows: return "Administrator"; + default: return die("initAdminUsername: invalid OS: "+CURRENT_OS); + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/system/Sleep.java b/src/main/java/org/cobbzilla/util/system/Sleep.java new file mode 100644 index 0000000..d11c86d --- /dev/null +++ b/src/main/java/org/cobbzilla/util/system/Sleep.java @@ -0,0 +1,54 @@ +package org.cobbzilla.util.system; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class Sleep { + + /** + * sleep for 100ms and throw an exception if interrupted + */ + public static void sleep () { sleep(100); } + + /** + * sleep and throw an exception if interrupted + * @param millis how long to sleep + */ + public static void sleep (long millis) { sleep(millis, "no reason for sleep given"); } + + /** + * sleep and throw an exception if interrupted + * @param millis how long to sleep + * @param reason something to add to the log statement if we are interrupted + */ + public static void sleep (long millis, String reason) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + throw new IllegalStateException("sleep interrupted (" + reason + ")"); + } + } + + /** + * A nap is something that you might get interrupted doing. + * @param millis how long to nap + * @return true if you napped without being interrupted, false if you were interrupted + */ + public static boolean nap (long millis) { return nap(millis, "no reason for nap given"); } + + /** + * A nap is something that you might get interrupted doing. + * @param millis how long to nap + * @param reason something to add to the log statement if we are interrupted + * @return true if you napped without being interrupted, false if you were interrupted + */ + public static boolean nap (long millis, String reason) { + try { + Thread.sleep(millis); + return true; + } catch (InterruptedException e) { + log.info("nap ("+reason+"): interrupted"); + return false; + } + } +} diff --git a/src/main/java/org/cobbzilla/util/system/WindowsAdminUtil.java b/src/main/java/org/cobbzilla/util/system/WindowsAdminUtil.java new file mode 100644 index 0000000..8a2d313 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/system/WindowsAdminUtil.java @@ -0,0 +1,20 @@ +package org.cobbzilla.util.system; + +import com.sun.jna.LastErrorException; +import com.sun.jna.Native; +import com.sun.jna.Platform; +import com.sun.jna.win32.StdCallLibrary; + +public class WindowsAdminUtil { + + public interface Shell32 extends StdCallLibrary { + boolean IsUserAnAdmin() throws LastErrorException; + } + + public static final Shell32 INSTANCE = Platform.isWindows() + ? (Shell32) Native.loadLibrary("shell32", Shell32.class) : null; + + public static boolean isUserWindowsAdmin() { + return INSTANCE != null && INSTANCE.IsUserAnAdmin(); + } +} \ No newline at end of file diff --git a/src/main/java/org/cobbzilla/util/time/ClockProvider.java b/src/main/java/org/cobbzilla/util/time/ClockProvider.java new file mode 100644 index 0000000..bed2b70 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/time/ClockProvider.java @@ -0,0 +1,12 @@ +package org.cobbzilla.util.time; + +import org.cobbzilla.util.daemon.ZillaRuntime; + +public interface ClockProvider { + + long now (); + + ClockProvider SYSTEM = new ClockProvider() { @Override public long now() { return System.currentTimeMillis(); } }; + ClockProvider ZILLA = new ClockProvider() { @Override public long now() { return ZillaRuntime.now(); } }; + +} diff --git a/src/main/java/org/cobbzilla/util/time/CurrentTime.java b/src/main/java/org/cobbzilla/util/time/CurrentTime.java new file mode 100644 index 0000000..e243f17 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/time/CurrentTime.java @@ -0,0 +1,37 @@ +package org.cobbzilla.util.time; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.cobbzilla.util.daemon.ZillaRuntime; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +@NoArgsConstructor +public class CurrentTime { + + @Getter @Setter private String zone; + @Getter @Setter private CurrentTimeValues now; + @Getter @Setter private CurrentTimeValues realNow; + + public CurrentTime(DateTimeZone tz) { + zone = tz.getID(); + now = new CurrentTimeValues(tz, ZillaRuntime.now()); + realNow = ZillaRuntime.getSystemTimeOffset() == 0 ? null : new CurrentTimeValues(tz, ZillaRuntime.realNow()); + } + + @NoArgsConstructor + public static class CurrentTimeValues { + @Getter @Setter private long now; + @Getter @Setter private String yyyyMMdd; + @Getter @Setter private String yyyyMMddHHmmss; + + public CurrentTimeValues(DateTimeZone tz, long now) { + this.now = now; + final DateTime time = new DateTime(now, tz); + yyyyMMdd = TimeUtil.DATE_FORMAT_YYYY_MM_DD.print(time); + yyyyMMddHHmmss = TimeUtil.DATE_FORMAT_YYYY_MM_DD_HH_mm_ss.print(time); + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/time/DefaultTimezone.java b/src/main/java/org/cobbzilla/util/time/DefaultTimezone.java new file mode 100644 index 0000000..042549b --- /dev/null +++ b/src/main/java/org/cobbzilla/util/time/DefaultTimezone.java @@ -0,0 +1,30 @@ +package org.cobbzilla.util.time; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.joda.time.DateTimeZone; + +import static org.cobbzilla.util.io.StreamUtil.stream2string; + +@Slf4j +public class DefaultTimezone { + + public static final String DEFAULT_TIMEZONE = "US/Eastern"; + + @Getter(lazy=true) private static final DateTimeZone zone = initTimeZone(); + + private static DateTimeZone initTimeZone() { + // first line that does not start with '#' within 'timezone.txt' resource file will be used + try { + final String[] lines = stream2string("timezone.txt").split("\n"); + for (String line : lines) if (!line.trim().startsWith("#")) return DateTimeZone.forID(line.trim()); + log.warn("initTimeZone: error, timezone.txt resource did not contain a valid timezone line, using default: "+DEFAULT_TIMEZONE); + return DateTimeZone.forID(DEFAULT_TIMEZONE); + + } catch (Exception e) { + log.warn("initTimeZone: error, returning default ("+DEFAULT_TIMEZONE+"): "+e.getClass().getSimpleName()+": "+e.getMessage()); + return DateTimeZone.forID(DEFAULT_TIMEZONE); + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/time/ImprovedTimezone.java b/src/main/java/org/cobbzilla/util/time/ImprovedTimezone.java new file mode 100644 index 0000000..0c6ce71 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/time/ImprovedTimezone.java @@ -0,0 +1,162 @@ +package org.cobbzilla.util.time; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.io.StreamUtil; +import org.cobbzilla.util.string.StringUtil; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.*; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; + +@Slf4j +public class ImprovedTimezone { + + @Getter private int id; + @Getter private String gmtOffset; + @Getter private String displayName; + @Getter private String linuxName; + @Getter private TimeZone timezone; + @Getter private String displayNameWithOffset; + + private static List TIMEZONES = null; + private static Map TIMEZONES_BY_ID = new HashMap<>(); + private static Map TIMEZONES_BY_GMT = new HashMap<>(); + private static Map TIMEZONES_BY_JNAME = new HashMap<>(); + private static final String TZ_FILE = StringUtil.packagePath(ImprovedTimezone.class) +"/timezones.txt"; + + private static TimeZone SYSTEM_TIMEZONE; + static { + try { + init(); + } catch (IOException e) { + String msg = "Error initializing ImprovedTimezone from timezones.txt: "+e; + log.error(msg, e); + die(msg, e); + } + final TimeZone sysTimezone = TimeZone.getDefault(); + ImprovedTimezone tz = TIMEZONES_BY_JNAME.get(sysTimezone.getDisplayName()); + if (tz == null) { + for (String displayName: TIMEZONES_BY_JNAME.keySet()) { + ImprovedTimezone tz1 = TIMEZONES_BY_JNAME.get(displayName); + String dn = displayName.replace("GMT-0","GMT-"); + dn = dn.replace("GMT+0", "GMT+"); + if (tz1.getGmtOffset().equals(dn)) { + tz = tz1; + break; + } + } + } + if (tz == null) { + throw new ExceptionInInitializerError("System Timezone could not be located in timezones.txt"); + } + + SYSTEM_TIMEZONE = tz.getTimezone(); + log.info("System Time Zone set to " + SYSTEM_TIMEZONE.getDisplayName()); + } + + private ImprovedTimezone (int id, + String gmtOffset, + TimeZone timezone, + String displayName, + String linuxName) { + this.id = id; + this.gmtOffset = gmtOffset; + this.timezone = timezone; + this.displayName = displayName; + this.linuxName = (linuxName == null) ? timezone.getDisplayName() : linuxName; + this.displayNameWithOffset = "("+gmtOffset+") "+displayName; + } + + public long getLocalTime (long systemTime) { + // convert time to GMT + final long gmtTime = systemTime - SYSTEM_TIMEZONE.getRawOffset(); + + // now that we're in GMT, convert to local + return gmtTime + getTimezone().getRawOffset(); + } + + public String toString () { + return "[ImprovedTimezone id="+id+" offset="+gmtOffset + +" name="+displayName+" zone="+timezone.getDisplayName() +"]"; + } + + public static List getTimeZones () { + return TIMEZONES; + } + + public static ImprovedTimezone getTimeZoneById (int id) { + final ImprovedTimezone tz = TIMEZONES_BY_ID.get(id); + if (tz == null) { + throw new IllegalArgumentException("Invalid timezone id: "+id); + } + return tz; + } + + public static ImprovedTimezone getTimeZoneByJavaDisplayName (String name) { + final ImprovedTimezone tz = TIMEZONES_BY_JNAME.get(name); + if (tz == null) { + throw new IllegalArgumentException("Invalid timezone name: "+name); + } + return tz; + } + + public static ImprovedTimezone getTimeZoneByGmtOffset(String value) { + return TIMEZONES_BY_GMT.get(value); + } + + /** + * Initialize timezones from a file on classpath. + * The first line of the file is a header that is ignored. + */ + private static void init () throws IOException { + + TIMEZONES = new ArrayList<>(); + try (InputStream in = StreamUtil.loadResourceAsStream(TZ_FILE)) { + if (in == null) { + throw new IOException("Error loading timezone file from classpath: "+TZ_FILE); + } + try (BufferedReader r = new BufferedReader(new InputStreamReader(in))) { + String line = r.readLine(); + while (line != null) { + line = r.readLine(); + if (line == null) break; + final ImprovedTimezone improvedTimezone = initZone(line); + TIMEZONES.add(improvedTimezone); + TIMEZONES_BY_ID.put(improvedTimezone.getId(), improvedTimezone); + TIMEZONES_BY_JNAME.put(improvedTimezone.getTimezone().getDisplayName(), improvedTimezone); + TIMEZONES_BY_GMT.put(improvedTimezone.getGmtOffset(), improvedTimezone); + } + } + } + } + private static ImprovedTimezone initZone (String line) { + try { + final StringTokenizer st = new StringTokenizer(line, "|"); + int id = Integer.parseInt(st.nextToken()); + final String gmtOffset = st.nextToken(); + final String timezoneName = st.nextToken(); + final String displayName = st.nextToken(); + final String linuxName = st.hasMoreTokens() ? st.nextToken() : timezoneName; + final TimeZone tz = TimeZone.getTimeZone(timezoneName); + if (!gmtOffset.equals("GMT") && isGMT(tz)) { + String msg = "Error looking up timezone: " + timezoneName + ": got GMT, expected " + gmtOffset; + log.error(msg); + die(msg); + } + return new ImprovedTimezone(id, gmtOffset, tz, displayName, linuxName); + + } catch (Exception e) { + return die("Error processing line: "+line+": "+e, e); + } + } + + private static boolean isGMT(TimeZone tz) { + return tz.getRawOffset() == 0; + } + +} diff --git a/src/main/java/org/cobbzilla/util/time/JavaTimezone.java b/src/main/java/org/cobbzilla/util/time/JavaTimezone.java new file mode 100644 index 0000000..123209d --- /dev/null +++ b/src/main/java/org/cobbzilla/util/time/JavaTimezone.java @@ -0,0 +1,56 @@ +package org.cobbzilla.util.time; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.util.*; +import java.util.stream.Collectors; + +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.io.StreamUtil.loadResourceAsStringOrDie; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.string.StringUtil.getPackagePath; + +@AllArgsConstructor +public class JavaTimezone { + + @Getter private final TimeZone timeZone; + + @Getter(lazy=true) private static final List allJavaTimezones = initAllTimezones(); + private static List initAllTimezones() { + return Arrays.stream(TimeZone.getAvailableIDs()).map(tz -> new JavaTimezone(TimeZone.getTimeZone(tz))).collect(Collectors.toList()); + } + + @Getter(lazy=true) private static final Map javaTimezoneMap = initTimezoneMap(); + private static Map initTimezoneMap() { + final Map map = getAllJavaTimezones().stream().collect(toMap(tz -> tz.getTimeZone().getID(), identity())); + final String path = getPackagePath(UnicodeTimezone.class) + "/java-timezone-exceptions.json"; + final Map exceptions = json(loadResourceAsStringOrDie(path), LinkedHashMap.class); + for (Map.Entry exc : exceptions.entrySet()) { + if (!map.containsKey(exc.getValue())) return die("initTimezoneMap: error reading "+path+": JavaTimezone not found: "+exc.getValue()); + map.put(exc.getKey(), map.get(exc.getValue())); + } + return map; + } + + public static JavaTimezone fromUnicode(UnicodeTimezone utz) { + final Map map = getJavaTimezoneMap(); + for (String alias : utz.aliases()) { + if (map.containsKey(alias)) return map.get(alias); + } + return die("fromUnicode: no JavaTimezone found for: "+utz); + } + + public static JavaTimezone fromLinux(LinuxTimezone ltz) { + final JavaTimezone javaTimezone = getJavaTimezoneMap().get(ltz.getName()); + if (javaTimezone == null) { + return die("fromLinux: no JavaTimezone found for: "+ltz.getName()); + } + return javaTimezone; + } + + public static JavaTimezone fromString(String value) { return getJavaTimezoneMap().get(value); } + +} diff --git a/src/main/java/org/cobbzilla/util/time/LinuxTimezone.java b/src/main/java/org/cobbzilla/util/time/LinuxTimezone.java new file mode 100644 index 0000000..75c0ac4 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/time/LinuxTimezone.java @@ -0,0 +1,55 @@ +package org.cobbzilla.util.time; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.io.StreamUtil.loadResourceAsStringOrDie; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.string.StringUtil.getPackagePath; +import static org.cobbzilla.util.system.CommandShell.execScript; + +@AllArgsConstructor +public class LinuxTimezone { + + @Getter private final String name; + + public JavaTimezone toJava () { return JavaTimezone.fromLinux(this); } + + @Getter(lazy=true) private static final List allLinuxTimezones = initAllTimezones(); + private static List initAllTimezones() { + final String[] lines = execScript("timedatectl list-timezones").split("\n"); + if (empty(lines)) die("initAllTimezones: error running timedatectl list-timezones, no output found on stdout"); + return Arrays.stream(lines).map(LinuxTimezone::new).collect(Collectors.toList()); + } + + @Getter(lazy=true) private static final Map linuxTimezoneMap = initTimezoneMap(); + private static Map initTimezoneMap() { + final Map map = getAllLinuxTimezones().stream().collect(toMap(LinuxTimezone::getName, identity())); + final String path = getPackagePath(UnicodeTimezone.class) + "/linux-timezone-exceptions.json"; + final Map exceptions = json(loadResourceAsStringOrDie(path), LinkedHashMap.class); + for (Map.Entry exc : exceptions.entrySet()) { + if (!map.containsKey(exc.getValue())) return die("initTimezoneMap: error reading "+path+": LinuxTimezone not found: "+exc.getValue()); + map.put(exc.getKey(), map.get(exc.getValue())); + } + return map; + } + + public static LinuxTimezone fromUnicode(UnicodeTimezone utz) { + final Map map = getLinuxTimezoneMap(); + for (String alias : utz.aliases()) { + if (map.containsKey(alias)) return map.get(alias); + } + return die("fromUnicode: no LinuxTimezone found for: "+utz); + } + +} diff --git a/src/main/java/org/cobbzilla/util/time/TimePeriod.java b/src/main/java/org/cobbzilla/util/time/TimePeriod.java new file mode 100644 index 0000000..75acca4 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/time/TimePeriod.java @@ -0,0 +1,25 @@ +package org.cobbzilla.util.time; + +import com.fasterxml.jackson.annotation.JsonCreator; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; + +public enum TimePeriod { + + seconds, minutes, hours, days, weeks, months, years; + + @JsonCreator public static TimePeriod fromString(String name) { + if (name == null) return die("TimePeriod: null name"); + switch (name.toLowerCase()) { + case "second": case "seconds": case "s": return seconds; + case "minute": case "minutes": case "m": return minutes; + case "hour": case "hours": case "h": return hours; + case "day": case "days": case "d": return days; + case "week": case "weeks": case "w": return weeks; + case "month": case "months": case "M": return months; + case "year": case "years": case "y": case "Y": return years; + default: return die("TimePeriod: invalid name: "+name); + } + } + +} diff --git a/src/main/java/org/cobbzilla/util/time/TimePeriodType.java b/src/main/java/org/cobbzilla/util/time/TimePeriodType.java new file mode 100644 index 0000000..a4846cb --- /dev/null +++ b/src/main/java/org/cobbzilla/util/time/TimePeriodType.java @@ -0,0 +1,57 @@ +package org.cobbzilla.util.time; + +import com.fasterxml.jackson.annotation.JsonCreator; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.util.Arrays; + +import static org.cobbzilla.util.daemon.ZillaRuntime.now; +import static org.cobbzilla.util.time.TimeRelativeType.future; +import static org.cobbzilla.util.time.TimeRelativeType.past; +import static org.cobbzilla.util.time.TimeRelativeType.present; +import static org.cobbzilla.util.time.TimeSpecifier.*; +import static org.cobbzilla.util.time.TimeUtil.*; + +@Slf4j @AllArgsConstructor +public enum TimePeriodType { + + today (present, todaySpecifier(), tomorrowSpecifier()), + tomorrow (future, tomorrowSpecifier(), futureDaySpecifier(2)), + week_to_date (past, t -> startOfWeekMillis(), nowSpecifier()), + month_to_date (past, t -> startOfMonthMillis(), nowSpecifier()), + quarter_to_date (past, t -> startOfQuarterMillis(), nowSpecifier()), + year_to_date (past, t -> startOfYearMillis(), nowSpecifier()), + yesterday (past, yesterdaySpecifier(), todaySpecifier()), + previous_week (past, t -> lastWeekMillis(), t -> startOfWeekMillis()), + previous_month (past, t -> lastWeekMillis(), t -> startOfMonthMillis()), + previous_quarter (past, t -> lastQuarterMillis(), t -> startOfQuarterMillis()), + previous_year (past, t -> lastYearMillis(), t -> startOfYearMillis()); + + @Getter private TimeRelativeType type; + private TimeSpecifier startSpecifier; + private TimeSpecifier endSpecifier; + + @JsonCreator public static TimePeriodType fromString (String val) { + try { + return valueOf(val.toLowerCase()); + } catch (IllegalArgumentException e) { + log.warn("fromString("+val+"): invalid value, use one of: "+Arrays.toString(values())); + throw e; + } + } + + @Getter private static TimePeriodType[] pastTypes = { + today, week_to_date, month_to_date, quarter_to_date, year_to_date, + yesterday, previous_week, previous_month, previous_quarter, previous_year + }; + + @Getter private static TimePeriodType[] futureTypes = { + today, tomorrow + }; + + public long start() { return startSpecifier.get(now()); } + public long end () { return endSpecifier.get(now()); } + +} diff --git a/src/main/java/org/cobbzilla/util/time/TimeRelativeType.java b/src/main/java/org/cobbzilla/util/time/TimeRelativeType.java new file mode 100644 index 0000000..dd2e7f0 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/time/TimeRelativeType.java @@ -0,0 +1,11 @@ +package org.cobbzilla.util.time; + +import com.fasterxml.jackson.annotation.JsonCreator; + +public enum TimeRelativeType { + + past, present, future; + + @JsonCreator public static TimeRelativeType fromString (String val) { return valueOf(val.toLowerCase()); } + +} diff --git a/src/main/java/org/cobbzilla/util/time/TimeSpecifier.java b/src/main/java/org/cobbzilla/util/time/TimeSpecifier.java new file mode 100644 index 0000000..144de37 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/time/TimeSpecifier.java @@ -0,0 +1,21 @@ +package org.cobbzilla.util.time; + +import org.joda.time.DateTime; +import org.joda.time.DurationFieldType; + +import static org.cobbzilla.util.daemon.ZillaRuntime.now; + +public interface TimeSpecifier { + + long get(long t); + + static TimeSpecifier nowSpecifier() { return t -> now(); } + static TimeSpecifier todaySpecifier() { return t -> new DateTime(t, DefaultTimezone.getZone()).withTimeAtStartOfDay().getMillis(); } + + static TimeSpecifier pastDaySpecifier(int count) { return t -> new DateTime(t, DefaultTimezone.getZone()).withTimeAtStartOfDay().withFieldAdded(DurationFieldType.days(), -1 * count).getMillis(); } + static TimeSpecifier yesterdaySpecifier() { return pastDaySpecifier(1); } + + static TimeSpecifier futureDaySpecifier(int count) { return t -> new DateTime(t, DefaultTimezone.getZone()).withTimeAtStartOfDay().withFieldAdded(DurationFieldType.days(), count).getMillis(); } + static TimeSpecifier tomorrowSpecifier() { return futureDaySpecifier(1); } + +} diff --git a/src/main/java/org/cobbzilla/util/time/TimeUtil.java b/src/main/java/org/cobbzilla/util/time/TimeUtil.java new file mode 100644 index 0000000..551c3e5 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/time/TimeUtil.java @@ -0,0 +1,199 @@ +package org.cobbzilla.util.time; + +import org.cobbzilla.util.string.StringUtil; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.DurationFieldType; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.PeriodFormatter; +import org.joda.time.format.PeriodFormatterBuilder; + +import java.util.concurrent.TimeUnit; + +import static org.cobbzilla.util.daemon.ZillaRuntime.*; + +public class TimeUtil { + + public static final long DAY = TimeUnit.DAYS.toMillis(1); + public static final long HOUR = TimeUnit.HOURS.toMillis(1); + public static final long MINUTE = TimeUnit.MINUTES.toMillis(1); + public static final long SECOND = TimeUnit.SECONDS.toMillis(1); + + public static final DateTimeFormatter DATE_FORMAT_MMDDYYYY = DateTimeFormat.forPattern("MM/dd/yyyy"); + public static final DateTimeFormatter DATE_FORMAT_MMMM_D_YYYY = DateTimeFormat.forPattern("MMMM d, yyyy"); + public static final DateTimeFormatter DATE_FORMAT_YYYY_MM_DD = DateTimeFormat.forPattern("yyyy-MM-dd"); + public static final DateTimeFormatter DATE_FORMAT_YYYYMMDD = DateTimeFormat.forPattern("yyyyMMdd"); + public static final DateTimeFormatter DATE_FORMAT_MMM_DD_YYYY = DateTimeFormat.forPattern("MMM dd, yyyy"); + public static final DateTimeFormatter DATE_FORMAT_YYYY_MM_DD_HH = DateTimeFormat.forPattern("yyyy-MM-dd-HH"); + public static final DateTimeFormatter DATE_FORMAT_YYYY_MM_DD_HH_mm_ss = DateTimeFormat.forPattern("yyyy-MM-dd-HH-mm-ss"); + public static final DateTimeFormatter DATE_FORMAT_YYYYMMDDHHMMSS = DateTimeFormat.forPattern("yyyyMMddHHmmss"); + public static final DateTimeFormatter DATE_FORMAT_HYPHEN_MMDDYYYY = DateTimeFormat.forPattern("MM-dd-yyyy"); + public static final DateTimeFormatter DATE_FORMAT_EEE_DD_MMM_YYYY_HH_MM_SS_ZZZ = DateTimeFormat.forPattern("EEE, dd MMM yyyy HH:mm:ss zzz"); + public static final DateTimeFormatter DATE_FORMAT_IF_MODIFIED_SINCE = DATE_FORMAT_EEE_DD_MMM_YYYY_HH_MM_SS_ZZZ; + public static final DateTimeFormatter DATE_FORMAT_LAST_MODIFIED = DATE_FORMAT_IF_MODIFIED_SINCE; + + public static final DateTimeFormatter[] DATE_TIME_FORMATS = { + DATE_FORMAT_YYYY_MM_DD, DATE_FORMAT_YYYY_MM_DD, DATE_FORMAT_YYYYMMDD, + DATE_FORMAT_YYYY_MM_DD_HH_mm_ss, DATE_FORMAT_YYYYMMDDHHMMSS, + DATE_FORMAT_HYPHEN_MMDDYYYY, DATE_FORMAT_MMDDYYYY + }; + + // For now only m (months) and d (days) are supported + // Both have to be present at the same time in that same order, but the value for each can be 0 to exclude that one - e.g. 0m15d. + public static final PeriodFormatter PERIOD_FORMATTER = new PeriodFormatterBuilder() + .appendMonths().appendSuffix("m").appendDays().appendSuffix("d").toFormatter(); + + public static Long parse(String time, DateTimeFormatter formatter) { + return empty(time) ? null : formatter.parseDateTime(time).getMillis(); + } + + public static Long parse(String time, DateTimeFormatter formatter, DateTimeZone timeZone) { + return empty(time) ? null : formatter.withZone(timeZone).parseDateTime(time).getMillis(); + } + + public static Object parse(String val) { + for (DateTimeFormatter f : DATE_TIME_FORMATS) { + try { + return TimeUtil.parse(val, f); + } catch (Exception ignored) { + // noop + } + } + return null; + } + + public static Long parse(String val, DateTimeZone timeZone) { + for (DateTimeFormatter f : DATE_TIME_FORMATS) { + try { + return TimeUtil.parse(val, f, timeZone); + } catch (Exception ignored) { + // noop + } + } + return null; + } + + public static String format(Long time, DateTimeFormatter formatter) { + return time == null ? null : new DateTime(time).toString(formatter); + } + + public static String formatDurationFrom(long start) { + long duration = now() - start; + return formatDuration(duration); + } + + public static String formatDuration(long duration) { + final boolean negative = duration < 0; + if (negative) duration *= -1L; + final String prefix = negative ? "-" : ""; + + long days = 0, hours = 0, mins = 0, secs = 0, millis = 0; + + if (duration > DAY) { + days = duration/DAY; + duration -= days * DAY; + } + if (duration > HOUR) { + hours = duration/HOUR; + duration -= hours * HOUR; + } + if (duration > MINUTE) { + mins = duration/MINUTE; + duration -= mins * MINUTE; + } + if (duration > SECOND) { + secs = duration/SECOND; + } + millis = duration - secs * SECOND; + + if (days > 0) return prefix+String.format("%1$01dd %2$02d:%3$02d:%4$02d.%5$04d", days, hours, mins, secs, millis); + return prefix+String.format("%1$02d:%2$02d:%3$02d.%4$04d", hours, mins, secs, millis); + } + + public static long parseDuration(String duration) { + if (empty(duration)) return 0; + final long val = Long.parseLong(duration.length() > 1 ? StringUtil.chopSuffix(duration) : duration); + switch (duration.charAt(duration.length()-1)) { + case 's': return TimeUnit.SECONDS.toMillis(val); + case 'm': return TimeUnit.MINUTES.toMillis(val); + case 'h': return TimeUnit.HOURS.toMillis(val); + case 'd': return TimeUnit.DAYS.toMillis(val); + default: return val; + } + } + + public static long addYear (long time) { + return new DateTime(time).withFieldAdded(DurationFieldType.years(), 1).getMillis(); + } + + public static long add365days (long time) { + return new DateTime(time).withFieldAdded(DurationFieldType.days(), 365).getMillis(); + } + + public static String timestamp() { return timestamp(ClockProvider.ZILLA); } + + public static String timestamp(ClockProvider clock) { + final long now = clock.now(); + return DATE_FORMAT_YYYY_MM_DD.print(now)+"-"+hexnow(now); + } + + public static long startOfWeekMillis() { return startOfWeek().getMillis(); } + public static DateTime startOfWeek() { return startOfWeek(DefaultTimezone.getZone()); } + public static DateTime startOfWeek(DateTimeZone zone) { + final DateTime startOfToday = new DateTime(zone).withTimeAtStartOfDay(); + return startOfToday.withFieldAdded(DurationFieldType.days(), -1 * startOfToday.getDayOfWeek()); + } + + public static long startOfMonthMillis() { return startOfMonth().getMillis(); } + public static DateTime startOfMonth() { return startOfMonth(DefaultTimezone.getZone()); } + public static DateTime startOfMonth(DateTimeZone zone) { + final DateTime startOfToday = new DateTime(zone).withTimeAtStartOfDay(); + return startOfToday.withFieldAdded(DurationFieldType.days(), -1 * startOfToday.getDayOfMonth()); + } + + public static DateTime startOfQuarter(DateTime t) { + final int month = t.getMonthOfYear(); + if (month <= 3) return t.withMonthOfYear(1); + if (month <= 6) return t.withMonthOfYear(4); + if (month <= 9) return t.withMonthOfYear(7); + return t.withMonthOfYear(10); + } + + public static long startOfQuarterMillis() { return startOfQuarter().getMillis(); } + public static DateTime startOfQuarter() { return startOfQuarter(DefaultTimezone.getZone()); } + public static DateTime startOfQuarter(DateTimeZone zone) { return startOfQuarter(new DateTime(zone).withTimeAtStartOfDay()); } + + public static long startOfYearMillis() { return startOfYear().getMillis(); } + public static DateTime startOfYear() { return startOfYear(DefaultTimezone.getZone()); } + public static DateTime startOfYear(DateTimeZone zone) { return new DateTime(zone).withTimeAtStartOfDay().withMonthOfYear(1).withDayOfMonth(1); } + + public static long yesterdayMillis() { return yesterday().getMillis(); } + public static DateTime yesterday() { return yesterday(DefaultTimezone.getZone()); } + public static DateTime yesterday(DateTimeZone zone) { return new DateTime(zone).withTimeAtStartOfDay().withFieldAdded(DurationFieldType.days(), -1); } + + public static long lastWeekMillis() { return lastWeek().getMillis(); } + public static DateTime lastWeek() { return lastWeek(DefaultTimezone.getZone()); } + public static DateTime lastWeek(DateTimeZone zone) { + return new DateTime(zone).withTimeAtStartOfDay().withFieldAdded(DurationFieldType.days(), -7).withDayOfWeek(1); + } + + public static long lastMonthMillis() { return lastMonth().getMillis(); } + public static DateTime lastMonth() { return lastMonth(DefaultTimezone.getZone()); } + public static DateTime lastMonth(DateTimeZone zone) { + return new DateTime(zone).withTimeAtStartOfDay().withFieldAdded(DurationFieldType.months(), -1).withDayOfMonth(1); + } + + public static long lastQuarterMillis() { return lastQuarter().getMillis(); } + public static DateTime lastQuarter() { return lastQuarter(DefaultTimezone.getZone()); } + public static DateTime lastQuarter(DateTimeZone zone) { + return startOfQuarter(new DateTime(zone).withTimeAtStartOfDay().withFieldAdded(DurationFieldType.months(), -3)); + } + + public static long lastYearMillis() { return lastYear().getMillis(); } + public static DateTime lastYear() { return lastYear(DefaultTimezone.getZone()); } + public static DateTime lastYear(DateTimeZone zone) { + return new DateTime(zone).withTimeAtStartOfDay().withFieldAdded(DurationFieldType.years(), -1).withDayOfYear(1); + } + +} diff --git a/src/main/java/org/cobbzilla/util/time/UnicodeTimezone.java b/src/main/java/org/cobbzilla/util/time/UnicodeTimezone.java new file mode 100644 index 0000000..37988df --- /dev/null +++ b/src/main/java/org/cobbzilla/util/time/UnicodeTimezone.java @@ -0,0 +1,87 @@ +package org.cobbzilla.util.time; + +import com.fasterxml.jackson.dataformat.xml.XmlMapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import lombok.Getter; +import lombok.Setter; +import org.cobbzilla.util.string.StringUtil; + +import java.util.*; + +import static java.util.stream.Collectors.toCollection; +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.io.StreamUtil.loadResourceAsStream; +import static org.cobbzilla.util.string.StringUtil.getPackagePath; + +public class UnicodeTimezone implements Comparable { + + // we exclude these oddball timezones, since they have no good mapping to Linux timezones + public static final String ETC_GMT_PREFIX = "Etc/GMT"; + + @Getter @Setter private String name; + @Getter @Setter private String description; + @Getter @Setter private String alias; + @Getter @Setter private Boolean deprecated; + @Getter @Setter private String preferred; + @Getter @Setter private String since; + + public String firstAlias() { + return empty(alias) ? StringUtil.EMPTY : aliases()[0]; + } + + public String[] aliases() { + return empty(alias) ? StringUtil.EMPTY_ARRAY : alias.split("\\s+"); + } + + public boolean deprecated() { return deprecated != null && deprecated; } + + @Override public int compareTo(UnicodeTimezone other) { return firstAlias().compareTo(other.firstAlias()); } + + public LinuxTimezone toLinux () { return LinuxTimezone.fromUnicode(this); } + public JavaTimezone toJava () { return JavaTimezone.fromUnicode(this); } + + public static UnicodeTimezone fromString(String value) { return getUnicodeTimezoneMap().get(value); } + + @Getter(lazy=true) private static final Set allUnicodeTimezones = initAllTimezones(); + private static Set initAllTimezones () { + try { + final String path = getPackagePath(UnicodeTimezone.class) + "/unicode-timezones.xml"; + final UnicodeXmlDocument document = new XmlMapper().readValue(loadResourceAsStream(path), UnicodeXmlDocument.class); + return Arrays.stream(document.getKeyword().getKey().getType()) + .filter(tz -> !tz.deprecated() && !tz.firstAlias().startsWith(ETC_GMT_PREFIX)) + .collect(toCollection(TreeSet::new)); + } catch (Exception e) { + return die("initAllTimezones: "+e); + } + } + + @Getter(lazy=true) private static final Map unicodeTimezoneMap = initTimezoneMap(); + private static Map initTimezoneMap() { + final Map map = new LinkedHashMap<>(); + getAllUnicodeTimezones().stream().filter(tz -> !tz.deprecated()).forEach(tz -> { + for (String alias : tz.aliases()) map.put(alias, tz); + }); + return map; + } + + public static class UnicodeXmlDocument { + @Getter @Setter private UnicodeVersion version; + @Getter @Setter private UnicodeKeyword keyword; + } + public static class UnicodeVersion { + @Getter @Setter private String number; + } + public static class UnicodeKeyword { + @Getter @Setter private UnicodeKey key; + } + public static class UnicodeKey { + @Getter @Setter private String name; + @Getter @Setter private String description; + @Getter @Setter private String alias; + + @JacksonXmlElementWrapper(useWrapping=false) + @Getter @Setter private UnicodeTimezone[] type; + } + +} diff --git a/src/main/java/org/cobbzilla/util/xml/CommonEntityResolver.java b/src/main/java/org/cobbzilla/util/xml/CommonEntityResolver.java new file mode 100644 index 0000000..27d653e --- /dev/null +++ b/src/main/java/org/cobbzilla/util/xml/CommonEntityResolver.java @@ -0,0 +1,42 @@ +package org.cobbzilla.util.xml; + +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.io.StreamUtil; +import org.cobbzilla.util.string.StringUtil; +import org.xml.sax.EntityResolver; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +@Slf4j +public class CommonEntityResolver implements EntityResolver { + + private static final String COMMON_DTD_ROOT = StringUtil.getPackagePath(CommonEntityResolver.class); + private static final String[][] COMMON_ENTITIES = { + { "-//W3C//DTD XHTML 1.0 Transitional//EN", COMMON_DTD_ROOT+"/xhtml1-transitional.dtd" }, + { "-//W3C//ENTITIES Latin 1 for XHTML//EN", COMMON_DTD_ROOT+"/xhtml-lat1.ent"}, + { "-//W3C//ENTITIES Symbols for XHTML//EN", COMMON_DTD_ROOT+"/xhtml-symbol.ent"}, + { "-//W3C//ENTITIES Special for XHTML//EN", COMMON_DTD_ROOT+"/xhtml-special.ent"} + }; + private static Map COMMON_ENTITY_MAP = new HashMap(); + static { + for (String[] entity : COMMON_ENTITIES) { + COMMON_ENTITY_MAP.put(entity[0], entity[1]); + } + } + + public InputSource resolveEntity(String publicId, String systemId) throws SAXException, IOException { + String resource = COMMON_ENTITY_MAP.get(publicId); + if (resource == null) { + String msg = "resolveEntity(" + publicId + ", " + systemId + ") called, returning null"; + log.info(msg); + System.out.println(msg); + return null; + } + return new InputSource(StreamUtil.loadResourceAsStream(resource)); + } + +} diff --git a/src/main/java/org/cobbzilla/util/xml/ElementIdGenerator.java b/src/main/java/org/cobbzilla/util/xml/ElementIdGenerator.java new file mode 100644 index 0000000..98d572b --- /dev/null +++ b/src/main/java/org/cobbzilla/util/xml/ElementIdGenerator.java @@ -0,0 +1,45 @@ +package org.cobbzilla.util.xml; + +import lombok.Getter; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import java.util.concurrent.atomic.AtomicInteger; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +public class ElementIdGenerator { + + private String prefix = ""; + private AtomicInteger counter; + + public ElementIdGenerator () { this(1); } + public ElementIdGenerator (int start) { counter = new AtomicInteger(start); } + public ElementIdGenerator (String prefix) { this(prefix, 1); } + public ElementIdGenerator (String prefix, int start) { this.prefix = prefix; counter = new AtomicInteger(start); } + + public Element id(Element e) { + if (empty(e.getAttribute("id"))) e.setAttribute("id", prefix+counter.getAndIncrement()); + return e; + } + + public Element create(Document doc, String elementName) { return id(doc.createElement(elementName)); } + + public Element text(Document doc, String elementName, String text) { return text(doc, elementName, text, null); } + + public Element text(Document doc, String elementName, String text, Integer truncate) { + final Element element = create(doc, elementName); + + if (text == null) return element; + text = text.trim(); + if (text.length() == 0) return element; + + if (truncate != null && text.trim().length() > truncate) text = text.trim().substring(0, truncate); + element.appendChild(doc.createTextNode(text)); + return element; + } + + @Getter(lazy=true) private final XmlElementFunction idFunction = initIdFunction(); + private XmlElementFunction initIdFunction() { return this::id; } + +} diff --git a/src/main/java/org/cobbzilla/util/xml/ElementMatcher.java b/src/main/java/org/cobbzilla/util/xml/ElementMatcher.java new file mode 100644 index 0000000..9db0901 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/xml/ElementMatcher.java @@ -0,0 +1,7 @@ +package org.cobbzilla.util.xml; + +import org.w3c.dom.Element; + +public interface ElementMatcher { + boolean matches (Element e); +} diff --git a/src/main/java/org/cobbzilla/util/xml/ElementTransformer.java b/src/main/java/org/cobbzilla/util/xml/ElementTransformer.java new file mode 100644 index 0000000..01faf34 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/xml/ElementTransformer.java @@ -0,0 +1,9 @@ +package org.cobbzilla.util.xml; + +import org.w3c.dom.Element; + +public interface ElementTransformer { + + T transform (Element e); + +} diff --git a/src/main/java/org/cobbzilla/util/xml/TidyHandlebarsSpanMerger.java b/src/main/java/org/cobbzilla/util/xml/TidyHandlebarsSpanMerger.java new file mode 100644 index 0000000..061dd65 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/xml/TidyHandlebarsSpanMerger.java @@ -0,0 +1,144 @@ +package org.cobbzilla.util.xml; + +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.collection.mappy.MappyList; +import org.w3c.dom.Document; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import java.util.Map; + +import static org.cobbzilla.util.handlebars.HandlebarsUtil.HB_END; +import static org.cobbzilla.util.handlebars.HandlebarsUtil.HB_START; + +@Slf4j +public class TidyHandlebarsSpanMerger implements TidyHelper { + + public static final TidyHandlebarsSpanMerger instance = new TidyHandlebarsSpanMerger(); + + @Override public void process(Document doc) { + final MappyList toRemove = new MappyList<>(); + mergeSpans(doc, toRemove); + for (Map.Entry n : toRemove.entrySet()) { + n.getKey().removeChild(n.getValue()); + } + } + + protected void mergeSpans(Node parent, MappyList toRemove) { + + Node spanStart = null; + StringBuilder spanTemp = null; + NodeList childNodes = parent.getChildNodes(); + for (int i=0; i 0) { + setSpan(spanStart, spanTemp); + } + } + + private void setSpan(Node spanStart, StringBuilder spanTemp) { + if (spanTemp == null || spanStart == null || spanStart.getFirstChild() == null) return; + spanStart.getFirstChild().setNodeValue(spanTemp.toString()); + } + + public static String scrubHandlebars(String text) { + final StringBuilder b = new StringBuilder(); + int start = 0; + int pos = text.indexOf(HB_START, start); + while (pos != -1) { + b.append(text.substring(start, pos)).append(HB_START); + start = pos + 2; + int endPos = text.indexOf(HB_END, start); + if (endPos == -1) { + b.append(text.substring(start)); + return b.toString(); + } else { + b.append(scrubHtmlEntities(text.substring(start, endPos))).append(HB_END); + start = endPos + 2; + } + pos = text.indexOf(HB_START, start); + } + if (start != text.length()-1) b.append(text.substring(start)); + return b.toString(); + } + + public static String scrubHtmlEntities(String s) { + return s.replace("“", "\"").replace("”", "\"") + .replace("‘", "'").replace("’", "'"); + } + + private StringBuilder append(StringBuilder b, String s) { + if (s == null || s.length() == 0) return b; + return b.append(s); + } + + private String collectText(Node node) { + final StringBuilder b = new StringBuilder(); + if (node.hasChildNodes()) { + final NodeList childNodes = node.getChildNodes(); + for (int i=0; i 0) { + NamedNodeMap map = elt.getAttributes(); + Set found = new HashSet(); + Set toRemove = null; + for (int i=0; i(); + toRemove.add(attr); + } else { + found.add(attr.getNodeName()); + } + } + if (toRemove != null) { + for (Attr attr : toRemove) { + elt.removeAttributeNode(attr); + } + } + } + NodeList childNodes = elt.getChildNodes(); + for (int i=0; i toRemove = null; + NodeList childNodes = parent.getChildNodes(); + for (int i=0; i(); + toRemove.add(child); + } + } + if (toRemove != null) { + for (Node dead : toRemove) { + parent.removeChild(dead); + } + } + childNodes = parent.getChildNodes(); + for (int i=0; i xpaths = new HashMap<>(); + + public XPath2 (String... expressions) { + if (empty(expressions)) die("XPath2: no expressions"); + for (String expr : expressions) { + xpaths.put(expr, new Path(expr)); + } + } + + public Map firstMatches (String xml) { return firstMatches(new Doc(xml)); } + + public Map firstMatches (Doc doc) { + final Map matches = new HashMap<>(); + for (Map.Entry path : xpaths.entrySet()) { + final String match = path.getValue().firstMatch(doc); + if (!empty(match)) matches.put(path.getKey(), match); + } + return matches; + } + + public String firstMatch(String xml) { + switch (xpaths.size()) { + case 0: return die("firstMatch: no xpath expressions"); + default: return die("firstMatch: more than one xpath expression"); + case 1: + final Map matches = firstMatches(xml); + return empty(matches) ? null : matches.values().iterator().next(); + } + } + + public static class Path { + + @Getter private XPathExpression expr; + + public Path (String xpath) { + try { + expr = getXPath().compile(xpath); + } catch (Exception e) { + die("XPath2.Path: "+e, e); + } + } + + public String firstMatch (String xml) { return firstMatch(new Doc(xml)); } + + public String firstMatch(Doc doc) { + try { + final List matches = (List) expr.evaluate(doc.getDoc(), XPathConstants.NODESET); + if (empty(matches) || matches.get(0) == null) return null; + final NodeInfo line = (NodeInfo) matches.get(0); + return line.iterate().next().getStringValue(); + + } catch (Exception e) { + return die("firstMatch: "+e, e); + } + } + } + + public static class Doc { + @Getter private TreeInfo doc; + public Doc (String xml) { + final InputSource is = new InputSource(new ByteArrayInputStream(xml.getBytes())); + final SAXSource ss = new SAXSource(is); + final Configuration config = ((XPathFactoryImpl) getXpathFactory()).getConfiguration(); + try { + doc = config.buildDocumentTree(ss); + } catch (Exception e) { + die("XPath2.Doc: "+e, e); + } + } + } +} diff --git a/src/main/java/org/cobbzilla/util/xml/XPathUtil.java b/src/main/java/org/cobbzilla/util/xml/XPathUtil.java new file mode 100644 index 0000000..709132f --- /dev/null +++ b/src/main/java/org/cobbzilla/util/xml/XPathUtil.java @@ -0,0 +1,170 @@ +package org.cobbzilla.util.xml; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.apache.xpath.XPathAPI; +import org.apache.xpath.objects.XObject; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.Text; +import org.w3c.dom.traversal.NodeIterator; +import org.w3c.tidy.Tidy; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.*; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.xml.TidyUtil.createTidy; + +@Slf4j @NoArgsConstructor +public class XPathUtil { + + public static final String DOC_ROOT_XPATH = "/"; + + @Getter @Setter private Collection pathExpressions; + @Getter @Setter private Tidy tidy = null; + @Getter @Setter private boolean removeScripts = true; + + public XPathUtil (String expr) { this(new String[] { expr }, true); } + public XPathUtil (String expr, boolean useTidy) { this(new String[] { expr }, useTidy); } + + public XPathUtil(String[] exprs) { this(Arrays.asList(exprs), createTidy()); } + public XPathUtil(String[] exprs, boolean useTidy) { this(Arrays.asList(exprs), createTidy()); } + + public XPathUtil(Collection passThruXPaths) { this(passThruXPaths, createTidy()); } + + public XPathUtil(Collection exprs, Tidy tidy) { + this.pathExpressions = exprs; + this.tidy = tidy; + } + + public List getFirstMatchList(InputStream in) throws ParserConfigurationException, IOException, SAXException, TransformerException { + return applyXPaths(in).values().iterator().next(); + } + + public List getFirstMatchList(String xml) throws ParserConfigurationException, IOException, SAXException, TransformerException { + return applyXPaths(xml).values().iterator().next(); + } + + public Map getFirstMatchMap(InputStream in) throws ParserConfigurationException, IOException, SAXException, TransformerException { + final Map> matchMap = applyXPaths(in); + final Map firstMatches = new HashMap<>(); + for (String key : matchMap.keySet()) { + final List found = matchMap.get(key); + if (!found.isEmpty()) firstMatches.put(key, found.get(0).getTextContent()); + } + return firstMatches; + } + + public Node getFirstMatch(InputStream in) throws ParserConfigurationException, IOException, SAXException, TransformerException { + final List nodes = getFirstMatchList(in); + return empty(nodes) ? null : nodes.get(0); + } + + public String getFirstMatchText (InputStream in) throws ParserConfigurationException, TransformerException, SAXException, IOException { + return getFirstMatch(in).getTextContent(); + } + + public String getFirstMatchText (String xml) throws ParserConfigurationException, TransformerException, SAXException, IOException { + final Node match = getFirstMatch(new ByteArrayInputStream(xml.getBytes())); + return match == null ? null : match.getTextContent(); + } + + public List getStrings (InputStream in) throws IOException, SAXException, ParserConfigurationException, TransformerException { + final List results = new ArrayList<>(); + final Document doc = getDocument(in); + for (String xpath : this.pathExpressions) { + final XObject found = XPathAPI.eval(doc, xpath); + if (found != null) results.add(found.toString()); + } + return results; + } + + public Map> applyXPaths(String xml) throws ParserConfigurationException, TransformerException, SAXException, IOException { + return applyXPaths(new ByteArrayInputStream(xml.getBytes())); + } + + public Map> applyXPaths(InputStream in) throws ParserConfigurationException, IOException, SAXException, TransformerException { + final Document document = getDocument(in); + return applyXPaths(document, document); + } + + public Map> applyXPaths(Document document, Node node) throws TransformerException { + final Map> allFound = new HashMap<>(); + // Use the simple XPath API to select a nodeIterator. + // System.out.println("Querying DOM using "+pathExpression); + for (String xpath : this.pathExpressions) { + final List found = new ArrayList<>(); + NodeIterator nl = XPathAPI.selectNodeIterator(node, xpath); + + // Serialize the found nodes to System.out. + // System.out.println(""); + Node n; + while ((n = nl.nextNode())!= null) { + if (isTextNode(n)) { + // DOM may have more than one node corresponding to a + // single XPath text node. Coalesce all contiguous text nodes + // at this level + StringBuilder sb = new StringBuilder(n.getNodeValue()); + for ( + Node nn = n.getNextSibling(); + isTextNode(nn); + nn = nn.getNextSibling() + ) { + sb.append(nn.getNodeValue()); + } + Text textNode = document.createTextNode(sb.toString()); + found.add(textNode); + + } else { + found.add(n); + // serializer.transform(new DOMSource(n), new StreamResult(new OutputStreamWriter(System.out))); + } + // System.out.println(); + } + // System.out.println(""); + allFound.put(xpath, found); + } + return allFound; + } + + public Document getDocument(String xml) throws ParserConfigurationException, SAXException, IOException { + return getDocument(new ByteArrayInputStream(xml.getBytes())); + } + + public Document getDocument(InputStream in) throws ParserConfigurationException, SAXException, IOException { + InputStream inStream = in; + if (tidy != null) { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + TidyUtil.parse(tidy, in, out, removeScripts); + inStream = new ByteArrayInputStream(out.toByteArray()); + } + + final InputSource inputSource = new InputSource(inStream); + final DocumentBuilderFactory dfactory = DocumentBuilderFactory.newInstance(); + dfactory.setNamespaceAware(false); + dfactory.setValidating(false); + // dfactory.setExpandEntityReferences(true); + final DocumentBuilder documentBuilder = dfactory.newDocumentBuilder(); + documentBuilder.setEntityResolver(new CommonEntityResolver()); + return documentBuilder.parse(inputSource); + } + + /** Decide if the node is text, and so must be handled specially */ + public static boolean isTextNode(Node n) { + if (n == null) return false; + short nodeType = n.getNodeType(); + return nodeType == Node.CDATA_SECTION_NODE || nodeType == Node.TEXT_NODE; + } +} diff --git a/src/main/java/org/cobbzilla/util/xml/XmlElementFunction.java b/src/main/java/org/cobbzilla/util/xml/XmlElementFunction.java new file mode 100644 index 0000000..5912251 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/xml/XmlElementFunction.java @@ -0,0 +1,9 @@ +package org.cobbzilla.util.xml; + +import org.w3c.dom.Element; + +public interface XmlElementFunction { + + void apply (Element element); + +} diff --git a/src/main/java/org/cobbzilla/util/xml/XmlUtil.java b/src/main/java/org/cobbzilla/util/xml/XmlUtil.java new file mode 100644 index 0000000..dc0d792 --- /dev/null +++ b/src/main/java/org/cobbzilla/util/xml/XmlUtil.java @@ -0,0 +1,265 @@ +package org.cobbzilla.util.xml; + +import org.atteo.xmlcombiner.XmlCombiner; +import org.cobbzilla.util.string.StringUtil; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.io.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.system.Bytes.KB; + +public class XmlUtil { + + public static final String XML10_UTF8 = "\n"; + public static final String XML10_UTF8_REGEX = "(<\\?xml [^?]+\\?>)"; + public static final Pattern XML10_UTF8_PATTERN = Pattern.compile(XML10_UTF8_REGEX); + + public static OutputStream merge (OutputStream out, String... documents) { + try { + final XmlCombiner combiner = new XmlCombiner(); + for (String document : documents) { + combiner.combine(StringUtil.stream(document)); + } + combiner.buildDocument(out); + return out; + + } catch (Exception e) { + return die("merge: "+e, e); + } + } + + public static String merge (String... documents) { return merge((int) (32*KB), documents); } + + public static String merge (int bufsiz, String... documents) { + final ByteArrayOutputStream buffer = new ByteArrayOutputStream(bufsiz); + return merge(buffer, documents).toString(); + } + + public static String replaceElement (String document, String fromElement, String toElement) { + return document + .replaceAll("<\\s*"+fromElement+"([^>]*)>", "<"+toElement+"$1>") + .replaceAll("", "") + .replaceAll("<\\s*"+fromElement+"\\s*/>", "<"+toElement+"/>"); + } + + public static Element textElement(Document doc, String element, String text) { + final Element node = doc.createElement(element); + node.appendChild(doc.createTextNode(text)); + return node; + } + + public static Document readDocument (String xml) { + try { + final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + final DocumentBuilder builder = factory.newDocumentBuilder(); + final InputSource is = new InputSource(new StringReader(xml)); + return builder.parse(is); + } catch (Exception e) { + return die("readDocument: "+e, e); + } + } + + public static void writeDocument (Document doc, Writer writer) { + try { + final TransformerFactory tf = TransformerFactory.newInstance(); + final Transformer transformer = tf.newTransformer(); + transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + transformer.transform(new DOMSource(doc), new StreamResult(writer)); + } catch (Exception e) { + die("writeDocument: " + e, e); + } + } + + public static String writeDocument(Document doc) { + final StringWriter writer = new StringWriter(); + writeDocument(doc, writer); + return writer.getBuffer().toString(); + } + + public static Node applyRecursively (Element element, XmlElementFunction func) { + func.apply(element); + final NodeList childNodes = element.getChildNodes(); + if (childNodes != null && childNodes.getLength() > 0) { + for (int i = 0; i < childNodes.getLength(); i++) { + final Node item = childNodes.item(i); + if (item instanceof Element) applyRecursively((Element) item, func); + } + } + return element; + } + + public static List findElements(Document doc, final String name) { + final List found = new ArrayList<>(); + applyRecursively(doc.getDocumentElement(), new MatchNodeName(name, found)); + return found; + } + + public static List findElements(Element element, final String name) { + final List found = new ArrayList<>(); + applyRecursively(element, new MatchNodeName(name, found)); + return found; + } + + public static List findElements(Element element, final String name, final String content) { + if (content == null) return findElements(element, name); + final List found = new ArrayList<>(); + applyRecursively(element, new MatchNodeNameAndText(name, content, found)); + return found; + } + + public static List findElements(Element element, final String name, final Map conditions) { + if (empty(conditions)) return findElements(element, name); + final List found = new ArrayList<>(); + applyRecursively(element, new MatchNodeSubElements(name, conditions, found)); + return found; + } + + public static Element findUniqueElement(Document doc, String name) { + final List elements = findElements(doc, name); + if (empty(elements)) return null; + if (elements.size() > 1) return die("add: multiple "+name+" elements found"); + return elements.get(0); + } + + public static Element findFirstElement(Document doc, String name) { + final List elements = findElements(doc, name); + return empty(elements) ? null : elements.get(0); + } + + public static Element findFirstElement(Element e, String name) { + final List elements = findElements(e, name); + return empty(elements) ? null : elements.get(0); + } + + public static Element findFirstElement(Element e, String name, String content) { + final List elements = findElements(e, name, content); + return empty(elements) ? null : elements.get(0); + } + + public static Element findFirstElement(Element e, String name, Map conditions) { + final List elements = findElements(e, name, conditions); + return empty(elements) ? null : elements.get(0); + } + + public static T findLargest(Document doc, final ElementMatcher matcher, final ElementTransformer transformer) { + final AtomicReference largest = new AtomicReference<>(); + applyRecursively(doc.getDocumentElement(), element -> { + if (matcher.matches(element)) { + final T val = transformer.transform(element); + if (val != null) { + synchronized (largest) { + final T curVal = largest.get(); + if (curVal == null || ((Comparable) val).compareTo(curVal) > 0) largest.set(val); + } + } + } + }); + return largest.get(); + } + + public static void removeElements(Document doc, String name) { + for (Element e : XmlUtil.findElements(doc, name)) e.getParentNode().removeChild(e); + } + + public static boolean same(Node n1, Node n2) { + if (n1 == n2) return true; + if (!n1.getNodeName().equals(n2.getNodeName())) return false; + final String id1 = id(n1); + final String id2 = id(n2); + return id1 != null && id2 != null && id1.equals(id2); + } + + public static String id (Node n) { + return n.hasAttributes() ? n.getAttributes().getNamedItem("id").getTextContent() : null; + } + + public static Element getElementById(Document doc, final String id) { + final AtomicReference found = new AtomicReference<>(); + applyRecursively(doc.getDocumentElement(), element -> { + if (id.equals(id(element))) { + if (found.get() != null) die("multiple elements found with id="+id); + found.set(element); + } + }); + return found.get(); + } + + public static String stripXmlPreamble(String xml) { + return XML10_UTF8_PATTERN.matcher(xml).replaceAll(""); + } + + private static abstract class MatchNodeXMLFunction implements XmlElementFunction { + protected final String value; + protected final List found; + + public MatchNodeXMLFunction(String value, List found) { + if (value == null) die("Value cannot be null"); + this.value = value; + this.found = found; + } + + protected abstract boolean check(Element element); + + @Override public void apply(Element element) { + if (element != null && check(element)) found.add(element); + } + } + + private static class MatchNodeName extends MatchNodeXMLFunction { + public MatchNodeName(String value, List found) { super(value, found); } + + @Override protected boolean check(Element element) { + return value.equalsIgnoreCase(element.getNodeName()); + } + } + + public static class MatchNodeNameAndText extends MatchNodeName { + private final String text; + + public MatchNodeNameAndText(String name, String text, List found) { + super(name, found); + if (text == null) die("Text cannot be null"); + this.text = text.trim(); + } + + @Override protected boolean check(Element element) { + return super.check(element) && text.equals(element.getTextContent().trim()); + } + } + + public static class MatchNodeSubElements extends MatchNodeName { + private final Map conditions; + + public MatchNodeSubElements(String name, Map conditions, List found) { + super(name, found); + if (empty(conditions)) die("Conditions map cannot be null"); + this.conditions = conditions; + } + + @Override protected boolean check(Element element) { + if (!super.check(element)) return false; + for (Map.Entry condition : conditions.entrySet()) { + if (findFirstElement(element, condition.getKey(), condition.getValue()) == null) return false; + } + return true; + } + } +} diff --git a/src/main/java/org/cobbzilla/util/xml/main/XPath.java b/src/main/java/org/cobbzilla/util/xml/main/XPath.java new file mode 100644 index 0000000..eaf143b --- /dev/null +++ b/src/main/java/org/cobbzilla/util/xml/main/XPath.java @@ -0,0 +1,45 @@ +package org.cobbzilla.util.xml.main; + +import lombok.Cleanup; +import org.cobbzilla.util.xml.XPathUtil; +import org.w3c.dom.Node; + +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.io.FileInputStream; +import java.io.OutputStreamWriter; +import java.util.List; + +public class XPath { + + public static void main (String[] args) throws Exception { + + final String file = args[0]; + final String expr = args[1]; + + @Cleanup FileInputStream in = new FileInputStream(file); + + final Transformer serializer = TransformerFactory.newInstance().newTransformer(); + serializer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + + final XPathUtil xp = new XPathUtil(expr, true); + final List nodes = xp.applyXPaths(in).get(expr); + System.out.println("Found "+nodes.size()+" matching nodes:"); + for (int i=0; i timeout) { + console.log('checkExists: timeout waiting for '+path); + phantom.exit(2); + } + window.setTimeout(function () { + if (checkExists(path, start, timeout)) { + phantom.exit(0); + } + }, 250); +} + +// @see https://github.com/ariya/phantomjs/issues/12685 +// @see https://github.com/ariya/phantomjs/issues/12936 +// the 2 lines below essentially hack phantomjs into outputting the right size PDF for US 8.5x11in paper when run on +// a linux machine. It has to do with the DPI of some underlying software, perhaps there is a better way to make the adjustment. +// or we should allow caller to name a saved preset config. then we could add support for A4 paper/etc. +page.zoomFactor = 1.25; +page.paperSize = { width: '615px', height: '790px', margin: '30px' }; + +page.open(url, function(status) { + if (status !== 'success') { + console.log('Error loading ('+status+'): '+url); + phantom.exit(1); + + } else { + page.render(outFile); + } +}); diff --git a/src/main/resources/org/cobbzilla/util/javascript/standard_js_lib.js b/src/main/resources/org/cobbzilla/util/javascript/standard_js_lib.js new file mode 100644 index 0000000..01cb58a --- /dev/null +++ b/src/main/resources/org/cobbzilla/util/javascript/standard_js_lib.js @@ -0,0 +1,176 @@ +// returns length of array or collection +function len (thing) { + if (typeof thing == 'undefined' || thing == null) return 0; + try { + return thing.size(); + } catch (err) { + return thing.length; + } +} + +// returns item[0] of item['0'] from array or collection +function first (thing) { + return _get_element(thing, 0); +} + +// returns true if item is in arr array +function found (item, arr) { return (typeof arr != 'undefined') && arr != null && _findIndex(arr, function(o){return ''+o == item}) != -1; } + +// returns true if any element in items array is found in arr array +function found_any (items, arr) { + if (typeof items == 'undefined' || typeof arr == 'undefined' || items == null || arr == null) return false; + for (var i = 0; i < len(items); i++) if (found(_get_element(items, i), arr)) return true; + return false; +} + +// calculate a percent value, where percentage is between 0 and 100, and amount is some number +function pct (percentage, amount) { return (parseFloat(percentage) / 100.0) * parseFloat(amount); } + +// standard comparison functions +function gt (x, compare) { return x > compare; } +function ge (x, compare) { return x >= compare; } +function lt (x, compare) { return x < compare; } +function le (x, compare) { return x <= compare; } +function eq (x, compare) { return x == compare; } +function ne (x, compare) { return x != compare; } +function startsWith (x, compare) { return x.startsWith(compare); } + +function _inner_get_element(not_null_arr, field) { + var v = not_null_arr[field]; + if ((v === undefined || v === null) && typeof not_null_arr.get === 'function') return not_null_arr.get(field); + return v; +} + +function _get_element(arr, field) { + if (arr == null) return null; + var v = _inner_get_element(arr, field); + if ((v === undefined || v === null) && typeof field === 'number') return _inner_get_element(arr, '' + field); + return v; +} + +function match_object (field, value, comparison, found) { + return function (obj) { + if (typeof obj == 'undefined' || obj == null) return false; + var target = obj; + var path = field; + var dotPos = path.indexOf('.'); + var v; + while (dotPos != -1) { + var prop = path.substring(0, dotPos); + v = _get_element(target, prop); + if (!v) return false; + target = v; + path = path.substring(dotPos+1); + dotPos = path.indexOf('.'); + } + v = _get_element(target, path); + if (v && comparison(v, value)) { + if (typeof found != 'undefined') { + found.push(obj); + return false; + } else { + return true; + } + } + return false; + }; +} + +// function to find the first object in array that matches field==value +// field may contain embedded dots to navigate within each object element of the array +function find (arr, field, value, comparison) { + if (typeof comparison == 'undefined') comparison = eq; + return _find(arr, match_object(field, value, comparison)); + // arr.find(match_object(field, value, comparison)); +} + +function _find (arr, func) { + if ((typeof arr == 'undefined') || arr == null) return null; + for (var i = 0; i < len(arr); i++) { + var v = _get_element(arr, i); + if (func(v)) return v; + } + return null; +} + +function _findIndex (arr, func) { + if ((typeof arr == 'undefined') || arr == null) return null; + for (var i = 0; i < len(arr); i++) { + if (func(_get_element(arr, i))) return i; + } + return -1; +} + +function contains (arr, field, comparison, value) { + var found = find(arr, field, value, comparison); + return (typeof found !== 'undefined') && found !== null && found !== false; +} + +// function to find the all object in array that match field==value +// field may contain embedded dots to navigate within each object element of the array +function find_all (arr, field, value, comparison) { + if (typeof comparison == 'undefined') comparison = eq; + var found = []; + if ((typeof arr == 'undefined') || arr == null || len(arr) == 0) return found; + _find(arr, match_object(field, value, comparison, found)); + // arr.find(match_object(field, value, comparison, found)); + return found; +} + +// return sum total of array elements +function sum_total (arr, field) { + var hasField = (typeof field != 'undefined'); + var sum = 0; + for (var i = 0; i < len(arr); i++) { + var v = _get_element(arr, i); + sum += (hasField ? _get_element(v, field) : v); + } + return sum; +} + +// returns a function that: +// 1) applies itemFunc function to an item, 2) uses comparison function to compare the result against compareVal +function compare (itemFunc, comparison, compareVal) { + return function (item) { + if ((typeof item != 'undefined') && item != null) { + var val = itemFunc(item); + if ((typeof val != 'undefined') && val != null) { + return comparison(itemFunc(item), compareVal); + } + } + return false; + }; +} + +// return an itemFunc that treats item.field as a percentage, and multiplies it by total, and compares it against compareVal +function compare_pct (field, total, comparison, compareVal) { + return function (item) { + if ((typeof item == 'undefined') || item == null) return false; + var val = _get_element(item, field); + return (typeof val != 'undefined') && val != null && comparison(pct(val, total), compareVal); + } +} + +// apply itemFunc to each item in array arr. if any such invocation of itemFunc returns true, then this function returns true +function match_any (arr, itemFunc) { + if ((typeof arr == 'undefined') || arr == null || len(arr) == 0) return false; + // var found = arr.find(itemFunc); + var found = _find(arr, itemFunc); + return (typeof found != 'undefined') && found != null; +} + +// functions for rounding up/down to nearest multiple +function up (x, multiple) { return multiple * parseInt(Math.ceil(parseFloat(x)/parseFloat(multiple))); } +function down (x, multiple) { return multiple * parseInt(Math.floor(parseFloat(x)/parseFloat(multiple))); } + +// percentage difference between two numbers, as a floating-point number (1% == 1.0, 100% == 100.0, -5.4% == -5.4) +function pct_diff (x, y) { + var d1 = 100 * (x/y); + var d2 = 100 * (y/x); + return d1 > d2 ? d1 : d2; +} + +// return true if x is "close enough" to y, in terms of max_pct (default is 1%) +function is_close_enough (x, y, max_pct) { + return 100 - pct_diff(x, y) <= ((typeof max_pct == "undefined") ? 1.0 : max_pct); +} diff --git a/src/main/resources/org/cobbzilla/util/string/calc_diff.js b/src/main/resources/org/cobbzilla/util/string/calc_diff.js new file mode 100644 index 0000000..cf6323b --- /dev/null +++ b/src/main/resources/org/cobbzilla/util/string/calc_diff.js @@ -0,0 +1,22 @@ +var dmp = new diff_match_patch(); + +function diff(text1, text2, opts) { + dmp.Diff_Timeout = parseFloat(opts['timeout']); + dmp.Diff_EditCost = parseFloat(opts['editCost']); + + var ms_start = (new Date()).getTime(); + var d = dmp.diff_main(text1, text2); + var ms_end = (new Date()).getTime(); + + if (opts['semantic']) { + dmp.diff_cleanupSemantic(d); + } + if (opts['efficiency']) { + dmp.diff_cleanupEfficiency(d); + } + return dmp.diff_prettyHtml(d); + // return dmp.diff_text1(d); + //+ '
Time: ' + (ms_end - ms_start) / 1000 + 's'; +} + +diff(text1, text2, opts); \ No newline at end of file diff --git a/src/main/resources/org/cobbzilla/util/string/country_codes.txt b/src/main/resources/org/cobbzilla/util/string/country_codes.txt new file mode 100644 index 0000000..85241f7 --- /dev/null +++ b/src/main/resources/org/cobbzilla/util/string/country_codes.txt @@ -0,0 +1,249 @@ +AD +AE +AF +AG +AI +AL +AM +AO +AQ +AR +AS +AT +AU +AW +AX +AZ +BA +BB +BD +BE +BF +BG +BH +BI +BJ +BL +BM +BN +BO +BQ +BR +BS +BT +BV +BW +BY +BZ +CA +CC +CD +CF +CG +CH +CI +CK +CL +CM +CN +CO +CR +CU +CV +CW +CX +CY +CZ +DE +DJ +DK +DM +DO +DZ +EC +EE +EG +EH +ER +ES +ET +FI +FJ +FK +FM +FO +FR +GA +GB +GD +GE +GF +GG +GH +GI +GL +GM +GN +GP +GQ +GR +GS +GT +GU +GW +GY +HK +HM +HN +HR +HT +HU +ID +IE +IL +IM +IN +IO +IQ +IR +IS +IT +JE +JM +JO +JP +KE +KG +KH +KI +KM +KN +KP +KR +KW +KY +KZ +LA +LB +LC +LI +LK +LR +LS +LT +LU +LV +LY +MA +MC +MD +ME +MF +MG +MH +MK +ML +MM +MN +MO +MP +MQ +MR +MS +MT +MU +MV +MW +MX +MY +MZ +NA +NC +NE +NF +NG +NI +NL +NO +NP +NR +NU +NZ +OM +PA +PE +PF +PG +PH +PK +PL +PM +PN +PR +PS +PT +PW +PY +QA +RE +RO +RS +RU +RW +SA +SB +SC +SD +SE +SG +SH +SI +SJ +SK +SL +SM +SN +SO +SR +SS +ST +SV +SX +SY +SZ +TC +TD +TF +TG +TH +TJ +TK +TL +TM +TN +TO +TR +TT +TV +TW +TZ +UA +UG +UM +US +UY +UZ +VA +VC +VE +VG +VI +VN +VU +WF +WS +YE +YT +ZA +ZM +ZW \ No newline at end of file diff --git a/src/main/resources/org/cobbzilla/util/string/default_langs.json b/src/main/resources/org/cobbzilla/util/string/default_langs.json new file mode 100644 index 0000000..366634b --- /dev/null +++ b/src/main/resources/org/cobbzilla/util/string/default_langs.json @@ -0,0 +1,251 @@ +{ + "AD": [], + "AE": [], + "AF": [], + "AG": [], + "AI": [], + "AL": [], + "AM": [], + "AO": [], + "AQ": [], + "AR": [], + "AS": [], + "AT": [], + "AU": ["en"], + "AW": [], + "AX": [], + "AZ": [], + "BA": [], + "BB": [], + "BD": [], + "BE": [], + "BF": [], + "BG": [], + "BH": [], + "BI": [], + "BJ": [], + "BL": [], + "BM": [], + "BN": [], + "BO": [], + "BQ": [], + "BR": [], + "BS": [], + "BT": [], + "BV": [], + "BW": [], + "BY": [], + "BZ": [], + "CA": ["en", "fr"], + "CC": [], + "CD": [], + "CF": [], + "CG": [], + "CH": [], + "CI": [], + "CK": [], + "CL": [], + "CM": [], + "CN": [], + "CO": [], + "CR": [], + "CU": [], + "CV": [], + "CW": [], + "CX": [], + "CY": [], + "CZ": [], + "DE": ["de_DE"], + "DJ": [], + "DK": [], + "DM": [], + "DO": [], + "DZ": [], + "EC": [], + "EE": [], + "EG": [], + "EH": [], + "ER": [], + "ES": [], + "ET": [], + "FI": [], + "FJ": [], + "FK": [], + "FM": [], + "FO": [], + "FR": ["fr"], + "GA": [], + "GB": ["en"], + "GD": [], + "GE": [], + "GF": [], + "GG": [], + "GH": [], + "GI": [], + "GL": [], + "GM": [], + "GN": [], + "GP": [], + "GQ": [], + "GR": [], + "GS": [], + "GT": [], + "GU": [], + "GW": [], + "GY": [], + "HK": [], + "HM": [], + "HN": [], + "HR": [], + "HT": [], + "HU": [], + "ID": [], + "IE": ["en"], + "IL": [], + "IM": [], + "IN": [], + "IO": [], + "IQ": [], + "IR": [], + "IS": [], + "IT": [], + "JE": [], + "JM": [], + "JO": [], + "JP": [], + "KE": [], + "KG": [], + "KH": [], + "KI": [], + "KM": [], + "KN": [], + "KP": [], + "KR": [], + "KW": [], + "KY": [], + "KZ": [], + "LA": [], + "LB": [], + "LC": [], + "LI": [], + "LK": [], + "LR": [], + "LS": [], + "LT": [], + "LU": [], + "LV": [], + "LY": [], + "MA": [], + "MC": [], + "MD": [], + "ME": [], + "MF": [], + "MG": [], + "MH": [], + "MK": [], + "ML": [], + "MM": [], + "MN": [], + "MO": [], + "MP": [], + "MQ": [], + "MR": [], + "MS": [], + "MT": [], + "MU": [], + "MV": [], + "MW": [], + "MX": ["es"], + "MY": [], + "MZ": [], + "NA": [], + "NC": [], + "NE": [], + "NF": [], + "NG": [], + "NI": [], + "NL": [], + "NO": [], + "NP": [], + "NR": [], + "NU": [], + "NZ": [], + "OM": [], + "PA": [], + "PE": [], + "PF": [], + "PG": [], + "PH": [], + "PK": [], + "PL": [], + "PM": [], + "PN": [], + "PR": [], + "PS": [], + "PT": [], + "PW": [], + "PY": [], + "QA": [], + "RE": [], + "RO": [], + "RS": [], + "RU": [], + "RW": [], + "SA": [], + "SB": [], + "SC": [], + "SD": [], + "SE": [], + "SG": [], + "SH": [], + "SI": [], + "SJ": [], + "SK": [], + "SL": [], + "SM": [], + "SN": [], + "SO": [], + "SR": [], + "SS": [], + "ST": [], + "SV": [], + "SX": [], + "SY": [], + "SZ": [], + "TC": [], + "TD": [], + "TF": [], + "TG": [], + "TH": [], + "TJ": [], + "TK": [], + "TL": [], + "TM": [], + "TN": [], + "TO": [], + "TR": [], + "TT": [], + "TV": [], + "TW": [], + "TZ": [], + "UA": [], + "UG": [], + "UM": [], + "US": ["en", "es", "zh", "tl", "vt", "ar", "fr", "ko", "ro", "de"], + "UY": [], + "UZ": [], + "VA": [], + "VC": [], + "VE": [], + "VG": [], + "VI": [], + "VN": [], + "VU": [], + "WF": [], + "WS": [], + "YE": [], + "YT": [], + "ZA": [], + "ZM": [], + "ZW": [] +} diff --git a/src/main/resources/org/cobbzilla/util/string/default_locales.json b/src/main/resources/org/cobbzilla/util/string/default_locales.json new file mode 100644 index 0000000..969bfa1 --- /dev/null +++ b/src/main/resources/org/cobbzilla/util/string/default_locales.json @@ -0,0 +1,251 @@ +{ + "AD": [], + "AE": [], + "AF": [], + "AG": [], + "AI": [], + "AL": [], + "AM": [], + "AO": [], + "AQ": [], + "AR": [], + "AS": [], + "AT": [], + "AU": ["en_AU"], + "AW": [], + "AX": [], + "AZ": [], + "BA": [], + "BB": [], + "BD": [], + "BE": [], + "BF": [], + "BG": [], + "BH": [], + "BI": [], + "BJ": [], + "BL": [], + "BM": [], + "BN": [], + "BO": [], + "BQ": [], + "BR": [], + "BS": [], + "BT": [], + "BV": [], + "BW": [], + "BY": [], + "BZ": [], + "CA": ["en_CA", "fr_CA"], + "CC": [], + "CD": [], + "CF": [], + "CG": [], + "CH": [], + "CI": [], + "CK": [], + "CL": [], + "CM": [], + "CN": [], + "CO": [], + "CR": [], + "CU": [], + "CV": [], + "CW": [], + "CX": [], + "CY": [], + "CZ": [], + "DE": ["de_DE"], + "DJ": [], + "DK": [], + "DM": [], + "DO": [], + "DZ": [], + "EC": [], + "EE": [], + "EG": [], + "EH": [], + "ER": [], + "ES": [], + "ET": [], + "FI": [], + "FJ": [], + "FK": [], + "FM": [], + "FO": [], + "FR": ["fr_FR"], + "GA": [], + "GB": ["en_GB"], + "GD": [], + "GE": [], + "GF": [], + "GG": [], + "GH": [], + "GI": [], + "GL": [], + "GM": [], + "GN": [], + "GP": [], + "GQ": [], + "GR": [], + "GS": [], + "GT": [], + "GU": [], + "GW": [], + "GY": [], + "HK": [], + "HM": [], + "HN": [], + "HR": [], + "HT": [], + "HU": [], + "ID": [], + "IE": ["en_IE"], + "IL": [], + "IM": [], + "IN": [], + "IO": [], + "IQ": [], + "IR": [], + "IS": [], + "IT": [], + "JE": [], + "JM": [], + "JO": [], + "JP": [], + "KE": [], + "KG": [], + "KH": [], + "KI": [], + "KM": [], + "KN": [], + "KP": [], + "KR": [], + "KW": [], + "KY": [], + "KZ": [], + "LA": [], + "LB": [], + "LC": [], + "LI": [], + "LK": [], + "LR": [], + "LS": [], + "LT": [], + "LU": [], + "LV": [], + "LY": [], + "MA": [], + "MC": [], + "MD": [], + "ME": [], + "MF": [], + "MG": [], + "MH": [], + "MK": [], + "ML": [], + "MM": [], + "MN": [], + "MO": [], + "MP": [], + "MQ": [], + "MR": [], + "MS": [], + "MT": [], + "MU": [], + "MV": [], + "MW": [], + "MX": [], + "MY": [], + "MZ": [], + "NA": [], + "NC": [], + "NE": [], + "NF": [], + "NG": [], + "NI": [], + "NL": [], + "NO": [], + "NP": [], + "NR": [], + "NU": [], + "NZ": [], + "OM": [], + "PA": [], + "PE": [], + "PF": [], + "PG": [], + "PH": [], + "PK": [], + "PL": [], + "PM": [], + "PN": [], + "PR": [], + "PS": [], + "PT": [], + "PW": [], + "PY": [], + "QA": [], + "RE": [], + "RO": [], + "RS": [], + "RU": [], + "RW": [], + "SA": [], + "SB": [], + "SC": [], + "SD": [], + "SE": [], + "SG": [], + "SH": [], + "SI": [], + "SJ": [], + "SK": [], + "SL": [], + "SM": [], + "SN": [], + "SO": [], + "SR": [], + "SS": [], + "ST": [], + "SV": [], + "SX": [], + "SY": [], + "SZ": [], + "TC": [], + "TD": [], + "TF": [], + "TG": [], + "TH": [], + "TJ": [], + "TK": [], + "TL": [], + "TM": [], + "TN": [], + "TO": [], + "TR": [], + "TT": [], + "TV": [], + "TW": [], + "TZ": [], + "UA": [], + "UG": [], + "UM": [], + "US": ["en_US", "es_US"], + "UY": [], + "UZ": [], + "VA": [], + "VC": [], + "VE": [], + "VG": [], + "VI": [], + "VN": [], + "VU": [], + "WF": [], + "WS": [], + "YE": [], + "YT": [], + "ZA": [], + "ZM": [], + "ZW": [] +} diff --git a/src/main/resources/org/cobbzilla/util/string/diff_match_patch.js b/src/main/resources/org/cobbzilla/util/string/diff_match_patch.js new file mode 100644 index 0000000..7870f54 --- /dev/null +++ b/src/main/resources/org/cobbzilla/util/string/diff_match_patch.js @@ -0,0 +1,51 @@ +// from https://code.google.com/p/google-diff-match-patch/ +// available under Apache 2.0 license +(function(){function diff_match_patch(){this.Diff_Timeout=1;this.Diff_EditCost=4;this.Match_Threshold=0.5;this.Match_Distance=1E3;this.Patch_DeleteThreshold=0.5;this.Patch_Margin=4;this.Match_MaxBits=32} + diff_match_patch.prototype.diff_main=function(a,b,c,d){"undefined"==typeof d&&(d=0>=this.Diff_Timeout?Number.MAX_VALUE:(new Date).getTime()+1E3*this.Diff_Timeout);if(null==a||null==b)throw Error("Null input. (diff_main)");if(a==b)return a?[[0,a]]:[];"undefined"==typeof c&&(c=!0);var e=c,f=this.diff_commonPrefix(a,b);c=a.substring(0,f);a=a.substring(f);b=b.substring(f);var f=this.diff_commonSuffix(a,b),g=a.substring(a.length-f);a=a.substring(0,a.length-f);b=b.substring(0,b.length-f);a=this.diff_compute_(a, + b,e,d);c&&a.unshift([0,c]);g&&a.push([0,g]);this.diff_cleanupMerge(a);return a}; + diff_match_patch.prototype.diff_compute_=function(a,b,c,d){if(!a)return[[1,b]];if(!b)return[[-1,a]];var e=a.length>b.length?a:b,f=a.length>b.length?b:a,g=e.indexOf(f);return-1!=g?(c=[[1,e.substring(0,g)],[0,f],[1,e.substring(g+f.length)]],a.length>b.length&&(c[0][0]=c[2][0]=-1),c):1==f.length?[[-1,a],[1,b]]:(e=this.diff_halfMatch_(a,b))?(f=e[0],a=e[1],g=e[2],b=e[3],e=e[4],f=this.diff_main(f,g,c,d),c=this.diff_main(a,b,c,d),f.concat([[0,e]],c)):c&&100c);v++){for(var n=-v+r;n<=v-t;n+=2){var l=g+n,m;m=n==-v||n!=v&&j[l-1]d)t+=2;else if(s>e)r+=2;else if(q&&(l=g+k-n,0<=l&&l= + u)return this.diff_bisectSplit_(a,b,m,s,c)}}for(n=-v+p;n<=v-w;n+=2){l=g+n;u=n==-v||n!=v&&i[l-1]d)w+=2;else if(m>e)p+=2;else if(!q&&(l=g+k-n,0<=l&&(l=u)))return this.diff_bisectSplit_(a,b,m,s,c)}}return[[-1,a],[1,b]]}; + diff_match_patch.prototype.diff_bisectSplit_=function(a,b,c,d,e){var f=a.substring(0,c),g=b.substring(0,d);a=a.substring(c);b=b.substring(d);f=this.diff_main(f,g,!1,e);e=this.diff_main(a,b,!1,e);return f.concat(e)}; + diff_match_patch.prototype.diff_linesToChars_=function(a,b){function c(a){for(var b="",c=0,f=-1,g=d.length;fd?a=a.substring(c-d):c=a.length?[h,j,n,l,g]:null}if(0>=this.Diff_Timeout)return null; + var d=a.length>b.length?a:b,e=a.length>b.length?b:a;if(4>d.length||2*e.lengthd[4].length?g:d:d:g;var j;a.length>b.length?(g=h[0],d=h[1],e=h[2],j=h[3]):(e=h[0],j=h[1],g=h[2],d=h[3]);h=h[4];return[g,d,e,j,h]}; + diff_match_patch.prototype.diff_cleanupSemantic=function(a){for(var b=!1,c=[],d=0,e=null,f=0,g=0,h=0,j=0,i=0;f=e){if(d>=b.length/2||d>=c.length/2)a.splice(f,0,[0,c.substring(0,d)]),a[f-1][1]=b.substring(0,b.length-d),a[f+1][1]=c.substring(d),f++}else if(e>=b.length/2||e>=c.length/2)a.splice(f,0,[0,b.substring(0,e)]),a[f-1][0]=1,a[f-1][1]=c.substring(0,c.length-e),a[f+1][0]=-1,a[f+1][1]=b.substring(e),f++;f++}f++}}; + diff_match_patch.prototype.diff_cleanupSemanticLossless=function(a){function b(a,b){if(!a||!b)return 6;var c=a.charAt(a.length-1),d=b.charAt(0),e=c.match(diff_match_patch.nonAlphaNumericRegex_),f=d.match(diff_match_patch.nonAlphaNumericRegex_),g=e&&c.match(diff_match_patch.whitespaceRegex_),h=f&&d.match(diff_match_patch.whitespaceRegex_),c=g&&c.match(diff_match_patch.linebreakRegex_),d=h&&d.match(diff_match_patch.linebreakRegex_),i=c&&a.match(diff_match_patch.blanklineEndRegex_),j=d&&b.match(diff_match_patch.blanklineStartRegex_); + return i||j?5:c||d?4:e&&!g&&h?3:g||h?2:e||f?1:0}for(var c=1;c=i&&(i=k,g=d,h=e,j=f)}a[c-1][1]!=g&&(g?a[c-1][1]=g:(a.splice(c-1,1),c--),a[c][1]= + h,j?a[c+1][1]=j:(a.splice(c+1,1),c--))}c++}};diff_match_patch.nonAlphaNumericRegex_=/[^a-zA-Z0-9]/;diff_match_patch.whitespaceRegex_=/\s/;diff_match_patch.linebreakRegex_=/[\r\n]/;diff_match_patch.blanklineEndRegex_=/\n\r?\n$/;diff_match_patch.blanklineStartRegex_=/^\r?\n\r?\n/; + diff_match_patch.prototype.diff_cleanupEfficiency=function(a){for(var b=!1,c=[],d=0,e=null,f=0,g=!1,h=!1,j=!1,i=!1;fb)break;e=c;f=d}return a.length!=g&&-1===a[g][0]?f:f+(b-e)}; + diff_match_patch.prototype.diff_prettyHtml=function(a){for(var b=[],c=/&/g,d=//g,f=/\n/g,g=0;g");switch(h){case 1:b[g]=''+j+"";break;case -1:b[g]=''+j+"";break;case 0:b[g]=""+j+""}}return b.join("")}; + diff_match_patch.prototype.diff_text1=function(a){for(var b=[],c=0;cthis.Match_MaxBits)throw Error("Pattern too long for this browser.");var e=this.match_alphabet_(b),f=this,g=this.Match_Threshold,h=a.indexOf(b,c);-1!=h&&(g=Math.min(d(0,h),g),h=a.lastIndexOf(b,c+b.length),-1!=h&&(g=Math.min(d(0,h),g)));for(var j=1<=i;p--){var w=e[a.charAt(p-1)];k[p]=0===t?(k[p+1]<<1|1)&w:(k[p+1]<<1|1)&w|((r[p+1]|r[p])<<1|1)|r[p+1];if(k[p]&j&&(w=d(t,p-1),w<=g))if(g=w,h=p-1,h>c)i=Math.max(1,2*c-h);else break}if(d(t+1,c)>g)break;r=k}return h}; + diff_match_patch.prototype.match_alphabet_=function(a){for(var b={},c=0;c=2*this.Patch_Margin&& + e&&(this.patch_addContext_(a,h),c.push(a),a=new diff_match_patch.patch_obj,e=0,h=d,f=g)}1!==i&&(f+=k.length);-1!==i&&(g+=k.length)}e&&(this.patch_addContext_(a,h),c.push(a));return c};diff_match_patch.prototype.patch_deepCopy=function(a){for(var b=[],c=0;cthis.Match_MaxBits){if(j=this.match_main(b,h.substring(0,this.Match_MaxBits),g),-1!=j&&(i=this.match_main(b,h.substring(h.length-this.Match_MaxBits),g+h.length-this.Match_MaxBits),-1==i||j>=i))j=-1}else j=this.match_main(b,h,g); + if(-1==j)e[f]=!1,d-=a[f].length2-a[f].length1;else if(e[f]=!0,d=j-g,g=-1==i?b.substring(j,j+h.length):b.substring(j,i+this.Match_MaxBits),h==g)b=b.substring(0,j)+this.diff_text2(a[f].diffs)+b.substring(j+h.length);else if(g=this.diff_main(h,g,!1),h.length>this.Match_MaxBits&&this.diff_levenshtein(g)/h.length>this.Patch_DeleteThreshold)e[f]=!1;else{this.diff_cleanupSemanticLossless(g);for(var h=0,k,i=0;ie[0][1].length){var f=b-e[0][1].length;e[0][1]=c.substring(e[0][1].length)+e[0][1];d.start1-=f;d.start2-=f;d.length1+=f;d.length2+=f}d=a[a.length-1];e=d.diffs;0==e.length||0!=e[e.length-1][0]?(e.push([0, + c]),d.length1+=b,d.length2+=b):b>e[e.length-1][1].length&&(f=b-e[e.length-1][1].length,e[e.length-1][1]+=c.substring(0,f),d.length1+=f,d.length2+=f);return c}; + diff_match_patch.prototype.patch_splitMax=function(a){for(var b=this.Match_MaxBits,c=0;c2*b?(h.length1+=i.length,e+=i.length,j=!1,h.diffs.push([g,i]),d.diffs.shift()):(i=i.substring(0,b-h.length1-this.Patch_Margin),h.length1+=i.length,e+=i.length,0===g?(h.length2+=i.length,f+=i.length):j=!1,h.diffs.push([g,i]),i==d.diffs[0][1]?d.diffs.shift():d.diffs[0][1]=d.diffs[0][1].substring(i.length))}g=this.diff_text2(h.diffs);g=g.substring(g.length-this.Patch_Margin);i=this.diff_text1(d.diffs).substring(0,this.Patch_Margin);""!==i&& + (h.length1+=i.length,h.length2+=i.length,0!==h.diffs.length&&0===h.diffs[h.diffs.length-1][0]?h.diffs[h.diffs.length-1][1]+=i:h.diffs.push([0,i]));j||a.splice(++c,0,h)}}};diff_match_patch.prototype.patch_toText=function(a){for(var b=[],c=0;c + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/org/cobbzilla/util/xml/xhtml-lat1.ent b/src/main/resources/org/cobbzilla/util/xml/xhtml-lat1.ent new file mode 100644 index 0000000..ffee223 --- /dev/null +++ b/src/main/resources/org/cobbzilla/util/xml/xhtml-lat1.ent @@ -0,0 +1,196 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/org/cobbzilla/util/xml/xhtml-special.ent b/src/main/resources/org/cobbzilla/util/xml/xhtml-special.ent new file mode 100644 index 0000000..ca358b2 --- /dev/null +++ b/src/main/resources/org/cobbzilla/util/xml/xhtml-special.ent @@ -0,0 +1,80 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/org/cobbzilla/util/xml/xhtml-symbol.ent b/src/main/resources/org/cobbzilla/util/xml/xhtml-symbol.ent new file mode 100644 index 0000000..63c2abf --- /dev/null +++ b/src/main/resources/org/cobbzilla/util/xml/xhtml-symbol.ent @@ -0,0 +1,237 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/org/cobbzilla/util/xml/xhtml1-transitional.dtd b/src/main/resources/org/cobbzilla/util/xml/xhtml1-transitional.dtd new file mode 100644 index 0000000..628f27a --- /dev/null +++ b/src/main/resources/org/cobbzilla/util/xml/xhtml1-transitional.dtd @@ -0,0 +1,1201 @@ + + + + + +%HTMLlat1; + + +%HTMLsymbol; + + +%HTMLspecial; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/test/java/org/cobbzilla/util/collection/ArrayUtilTest.java b/src/test/java/org/cobbzilla/util/collection/ArrayUtilTest.java new file mode 100644 index 0000000..90db0df --- /dev/null +++ b/src/test/java/org/cobbzilla/util/collection/ArrayUtilTest.java @@ -0,0 +1,42 @@ +package org.cobbzilla.util.collection; + +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.assertTrue; + +public class ArrayUtilTest { + + @Test public void testSlice () throws Exception { + + assertTrue(Arrays.deepEquals(ArrayUtil.slice( + new String[]{"one", "two", "three"}, 0, 2), + new String[]{"one", "two"})); + + assertTrue(Arrays.deepEquals(ArrayUtil.slice( + new String[]{"one", "two", "three"}, 0, 0), + new String[0])); + + assertTrue(Arrays.deepEquals(ArrayUtil.slice( + new String[]{"one", "two", "three"}, 2, 2), + new String[0])); + + assertTrue(Arrays.deepEquals(ArrayUtil.slice( + new String[]{"one", "two", "three"}, 3, 3), + new String[0])); + + assertTrue(Arrays.deepEquals(ArrayUtil.slice( + new String[]{"one", "two", "three"}, 0, 3), + new String[]{"one", "two", "three"})); + + assertTrue(Arrays.deepEquals(ArrayUtil.slice( + new String[]{"one", "two", "three"}, 1, 3), + new String[]{"two", "three"})); + + assertTrue(Arrays.deepEquals(ArrayUtil.slice( + new String[]{"one", "two", "three"}, 2, 3), + new String[]{"three"})); + + } +} diff --git a/src/test/java/org/cobbzilla/util/handlebars/HandlebarsUtilTest.java b/src/test/java/org/cobbzilla/util/handlebars/HandlebarsUtilTest.java new file mode 100644 index 0000000..a7fcd02 --- /dev/null +++ b/src/test/java/org/cobbzilla/util/handlebars/HandlebarsUtilTest.java @@ -0,0 +1,27 @@ +package org.cobbzilla.util.handlebars; + +import com.github.jknack.handlebars.Handlebars; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.cobbzilla.util.io.StreamUtil.stream2string; +import static org.cobbzilla.util.string.StringUtil.getPackagePath; +import static org.junit.Assert.assertEquals; + +public class HandlebarsUtilTest { + + @Test public void testAltChars () throws Exception { + final Map ctx = new HashMap<>(); + ctx.put("name", "TEST_NAME"); + final String template = stream2string(getPackagePath(getClass())+"/alt_test.txt.hbs"); + final String expected = stream2string(getPackagePath(getClass())+"/alt_test.txt"); + + final Handlebars hbs = new Handlebars(new HandlebarsUtil(getClass().getSimpleName())); + final String result = HandlebarsUtil.apply(hbs, template, ctx, '<', '>'); + + assertEquals("handlebars template processed incorrectly", expected, result); + } + +} diff --git a/src/test/java/org/cobbzilla/util/io/TarballTest.java b/src/test/java/org/cobbzilla/util/io/TarballTest.java new file mode 100644 index 0000000..0372b63 --- /dev/null +++ b/src/test/java/org/cobbzilla/util/io/TarballTest.java @@ -0,0 +1,79 @@ +package org.cobbzilla.util.io; + +import com.google.common.io.Files; +import org.apache.commons.io.FileUtils; +import org.cobbzilla.util.security.ShaUtil; +import org.cobbzilla.util.string.StringUtil; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import static org.cobbzilla.util.io.FileUtil.getDefaultTempDir; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TarballTest { + + private File tempDir; + + @Before public void createTempDir () throws Exception { tempDir = Files.createTempDir(); } + + @After public void deleteTempDir () throws Exception { FileUtils.deleteDirectory(tempDir); } + + @Test public void testUnrollAndReroll () throws Exception { + + // copy tarball from resources to temp file + final File tarball = StreamUtil.stream2temp(StreamUtil.loadResourceAsStream(StringUtil.packagePath(getClass()) + "/test.tar.gz")); + + // unroll the tarball + Tarball.unroll(tarball, tempDir); + + // list files in tempDir, we expect file1 and subdir + validateUnrolledTarball(tempDir); + + final File newTar = File.createTempFile("temp", ".tar", getDefaultTempDir()); + Tarball.roll(newTar, this.tempDir); + + // reset tempdir + deleteTempDir(); createTempDir(); + Tarball.unroll(tarball, tempDir); + + // re-validate, should still pass + validateUnrolledTarball(tempDir); + } + + protected void validateUnrolledTarball(File unrolledDir) throws IOException { + final File[] files = unrolledDir.listFiles(); + assertEquals(2, files.length); + + int fileIndex, subdirIndex; + if (files[0].getName().equals("file1.txt")) { + fileIndex = 0; + subdirIndex = 1; + } else { + subdirIndex = 0; + fileIndex = 1; + } + + String shasum; + + assertEquals("file1.txt", files[fileIndex].getName()); + assertEquals(4160, files[fileIndex].length()); + shasum = StringUtil.tohex(ShaUtil.sha256(FileUtil.toString(files[fileIndex]))); + assertEquals("62b74f00961184e2b448adfcf38ee6186d94b15ea6a364917d72645d8705a30c", shasum); + + assertEquals("subdir", files[subdirIndex].getName()); + assertTrue("subdir", files[subdirIndex].isDirectory()); + + final File[] subdirFiles = files[subdirIndex].listFiles(); + assertEquals(1, subdirFiles.length); + assertEquals("subfile.txt", subdirFiles[0].getName()); + assertEquals(65072, subdirFiles[0].length()); + shasum = StringUtil.tohex(ShaUtil.sha256(FileUtil.toString(subdirFiles[0]))); + assertEquals("4ded9b8ff2cb3b94ff0bb15b8e17aff795c7f64bd088d5d70acb38ad286e2d12", shasum); + } + +} diff --git a/src/test/java/org/cobbzilla/util/json/JsonEditTest.java b/src/test/java/org/cobbzilla/util/json/JsonEditTest.java new file mode 100644 index 0000000..6a34c20 --- /dev/null +++ b/src/test/java/org/cobbzilla/util/json/JsonEditTest.java @@ -0,0 +1,160 @@ +package org.cobbzilla.util.json; + +import org.cobbzilla.util.io.StreamUtil; +import org.cobbzilla.util.json.data.TestData; +import org.cobbzilla.util.string.StringUtil; +import org.junit.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Random; + +import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; +import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class JsonEditTest { + + public static final String TEST_JSON = StringUtil.getPackagePath(JsonEditTest.class)+"/test.json"; + private final Random random = new Random(); + + @Test public void testEditJson() throws Exception { + + JsonEdit jsonEdit; + String result; + + // replace a node (overwrite an object with an integer) + final Integer toReplace = random.nextInt(); + jsonEdit = new JsonEdit() + .setJsonData(testJson()) + .addOperation(new JsonEditOperation() + .setType(JsonEditOperationType.write) + .setPath("thing.field2") + .setJson(toReplace.toString())); + result = jsonEdit.edit(); + assertEquals(toReplace, JsonUtil.fromJson(result, "thing.field2", Integer.class)); + + // replace a node within an array + final Integer toReplace2 = random.nextInt(); + jsonEdit = new JsonEdit() + .setJsonData(testJson()) + .addOperation(new JsonEditOperation() + .setType(JsonEditOperationType.write) + .setPath("thing.field1[2]") + .setJson(toReplace2.toString())); + result = jsonEdit.edit(); + assertEquals(toReplace2, JsonUtil.fromJson(result, "thing.field1[2]", Integer.class)); + + // add a node + final String rand0 = randomAlphanumeric(10); + final String toAdd0 = "{\"sub1\": true, \"sub2\": \"" + rand0 + "\"}"; + jsonEdit = new JsonEdit() + .setJsonData(testJson()) + .addOperation(new JsonEditOperation() + .setType(JsonEditOperationType.write) + .setPath("thing.field3") + .setJson(toAdd0)); + result = jsonEdit.edit(); + assertEquals(rand0, JsonUtil.fromJson(result, "thing.field3.sub2", String.class)); + + // add a node + final String rand = randomAlphanumeric(10); + final String toAdd = "{\"subC\": \""+ rand + "\"}"; + jsonEdit = new JsonEdit() + .setJsonData(testJson()) + .addOperation(new JsonEditOperation() + .setType(JsonEditOperationType.write) + .setPath("thing.field2") + .setJson(toAdd)); + result = jsonEdit.edit(); + assertEquals(rand, JsonUtil.fromJson(result, "thing.field2.subC", String.class)); + + // add a node at the root + final String rand2 = randomAlphanumeric(10); + final String rootAdd = "{\"rootfoo\": \""+ rand2 + "\"}"; + jsonEdit = new JsonEdit() + .setJsonData(testJson()) + .addOperation(new JsonEditOperation() + .setType(JsonEditOperationType.write) + .setJson(rootAdd)); + result = jsonEdit.edit(); + assertEquals(rand2, JsonUtil.fromJson(result, "rootfoo", String.class)); + + // add a numeric node at the root + final Integer rootAdd2 = random.nextInt(); + jsonEdit = new JsonEdit() + .setJsonData(testJson()) + .addOperation(new JsonEditOperation() + .setType(JsonEditOperationType.write) + .setPath("newguy") + .setJson(rootAdd2.toString())); + result = jsonEdit.edit(); + assertEquals(rootAdd2, JsonUtil.fromJson(result, "newguy", Integer.class)); + + // replace something that doesn't exist -- should add it + final String rand3 = randomAlphanumeric(10); + final String toReplace3 = "\""+rand3+"\""; + jsonEdit = new JsonEdit() + .setJsonData(testJson()) + .addOperation(new JsonEditOperation() + .setType(JsonEditOperationType.write) + .setPath("thing.field3") + .setJson(toReplace3)); + result = jsonEdit.edit(); + assertEquals(rand3, JsonUtil.fromJson(result, "thing.field3", String.class)); + + // append to a list + final String rand4 = randomAlphanumeric(10); + final String toReplace4 = "\""+rand4+"\""; + jsonEdit = new JsonEdit() + .setJsonData(testJson()) + .addOperation(new JsonEditOperation() + .setType(JsonEditOperationType.write) + .setPath("thing.field1[]") + .setJson(toReplace4)); + result = jsonEdit.edit(); + assertEquals(rand4, JsonUtil.fromJson(result, "thing.field1[3]", String.class)); + assertEquals(4, JsonUtil.fromJson(result, "thing.field1", String[].class).length); + + // write a new, deep node at the root level of the tree. + // ObjectNodes should be created along the way for missing nodes. + final Integer deepNodeValue = random.nextInt(); + final String deepNodePath = "newRootField."+randomAlphabetic(4)+"."+randomAlphabetic(4); + jsonEdit = new JsonEdit() + .setJsonData(testJson()) + .addOperation(new JsonEditOperation() + .setType(JsonEditOperationType.write) + .setPath(deepNodePath) + .setJson(deepNodeValue.toString())); + result = jsonEdit.edit(); + assertEquals(deepNodeValue, JsonUtil.fromJson(result, deepNodePath, Integer.class)); + + // write a new, deep node within the tree. + // ObjectNodes should be created along the way for missing nodes. + final Integer deepNodeValue2 = random.nextInt(); + final String deepNodePath2 = "thing.newField."+randomAlphabetic(4)+"."+randomAlphabetic(4); + jsonEdit = new JsonEdit() + .setJsonData(testJson()) + .addOperation(new JsonEditOperation() + .setType(JsonEditOperationType.write) + .setPath(deepNodePath2) + .setJson(deepNodeValue2.toString())); + result = jsonEdit.edit(); + assertEquals(deepNodeValue2, JsonUtil.fromJson(result, deepNodePath2, Integer.class)); + + // delete a node + jsonEdit = new JsonEdit() + .setJsonData(testJson()) + .addOperation(new JsonEditOperation() + .setType(JsonEditOperationType.delete) + .setPath("thing.field2")); + result = jsonEdit.edit(); + assertNull(JsonUtil.fromJson(result, TestData.class).thing.field2); + } + + private InputStream testJson() throws IOException { + return StreamUtil.loadResourceAsStream(TEST_JSON); + } + +} diff --git a/src/test/java/org/cobbzilla/util/json/JsonUtilTest.java b/src/test/java/org/cobbzilla/util/json/JsonUtilTest.java new file mode 100644 index 0000000..be49a78 --- /dev/null +++ b/src/test/java/org/cobbzilla/util/json/JsonUtilTest.java @@ -0,0 +1,135 @@ +package org.cobbzilla.util.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.commons.lang3.RandomStringUtils; +import org.cobbzilla.util.io.FileUtil; +import org.cobbzilla.util.io.StreamUtil; +import org.cobbzilla.util.json.data.TestData; +import org.cobbzilla.util.string.StringUtil; +import org.junit.Test; + +import java.io.File; +import java.util.*; + +import static org.cobbzilla.util.io.FileUtil.getDefaultTempDir; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.json.JsonUtil.toJson; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class JsonUtilTest { + + public static final String PREFIX = StringUtil.getPackagePath(JsonUtilTest.class); + private static final String TEST_JSON = PREFIX +"/test.json"; + + private static final Object[][] TESTS = new Object[][] { + new Object[] { + "id", new ReplacementValue() { @Override public String getValue(TestData testData) { return testData.id; } } + }, + new Object[] { + "thing.field1[1]", new ReplacementValue() { @Override public String getValue(TestData testData) { return testData.thing.field1[1]; } } + }, + new Object[] { + "another_thing.field1", new ReplacementValue() { @Override public String getValue(TestData testData) { return testData.another_thing.field1; } } + }, + }; + + @Test + public void testReplaceJsonValue () throws Exception { + + final String testJson = StreamUtil.loadResourceAsString(TEST_JSON); + final String replacement = RandomStringUtils.randomAlphanumeric(10); + + for (Object[] test : TESTS) { + assertReplacementMade(testJson, replacement, (String) test[0], (ReplacementValue) test[1]); + } + } + + public void assertReplacementMade(String testJson, String replacement, String path, ReplacementValue value) throws Exception { + final ObjectNode doc = JsonUtil.replaceNode(testJson, path, replacement); + final File temp = File.createTempFile("JsonUtilTest", ".json", getDefaultTempDir()); + FileUtil.toFile(temp, toJson(doc)); + final TestData data = JsonUtil.fromJson(temp, TestData.class); + assertEquals(replacement, value.getValue(data)); + } + + public interface ReplacementValue { + String getValue(TestData testData); + } + + @Test public void testMerge () throws Exception { + final String orig = StreamUtil.stream2string(PREFIX + "/merge/test1_orig.json"); + final String request = StreamUtil.stream2string(PREFIX + "/merge/test1_request.json"); + final String expected = StreamUtil.stream2string(PREFIX + "/merge/test1_expected.json"); + assertTrue(jsonEquals(expected.replaceAll("\\p{javaSpaceChar}+", ""), JsonUtil.mergeJson(orig, request).replaceAll("\\p{javaSpaceChar}+", ""))); + } + + private boolean jsonEquals(String j1, String j2) { + if (j1 == null) return j2 == null; + if (j2 == null) return false; + + final JsonNode n1 = json(j1, JsonNode.class); + final JsonNode n2 = json(j2, JsonNode.class); + return jsonEquals(n1, n2); + } + + private boolean jsonEquals(JsonNode n1, JsonNode n2) { + if (!n1.getNodeType().equals(n2.getNodeType())) return false; + + switch (n1.getNodeType()) { + case ARRAY: + return jsonArrayEquals((ArrayNode) n1, (ArrayNode) n2); + case OBJECT: + return jsonObjectEquals((ObjectNode) n1, (ObjectNode) n2); + case NULL: + return n1.isNull() == n2.isNull(); + default: + return n1.textValue().equals(n2.textValue()); + } + } + + private boolean jsonObjectEquals(ObjectNode n1, ObjectNode n2) { + final Map n1fields = toMap(n1); + final Map n2fields = toMap(n2); + if (n1fields.size() != n2fields.size()) return false; + for (Map.Entry n1entry : n1fields.entrySet()) { + if (!jsonEquals(n1entry.getValue(), n2fields.get(n1entry.getKey()))) return false; + } + return true; + } + + public static Map toMap(ObjectNode node) { + final Map map = new HashMap<>(); + for (Iterator iter = node.fieldNames(); iter.hasNext(); ) { + final String name = iter.next(); + map.put(name, node.get(name)); + } + return map; + } + + public boolean jsonArrayEquals(ArrayNode n1, ArrayNode n2) { + if (n1.size() != n2.size()) return false; + if (n1.size() == 0) return true; + + final List n1elements = new ArrayList<>(); + final List n2elements = new ArrayList<>(); + for (int i=0; i RATE_PERCENT_MAPPING = new HashMap() {{ + put(999.99, 16.90); + put(1499.99, 16.85); + put(1999.99, 16.35); + put(2499.99, 15.85); + put(2999.99, 15.35); + put(3499.99, 14.85); + put(3999.99, 14.35); + put(4499.99, 13.85); + put(4999.99, 12.10); + put(9999.99, 11.30); + put(14999.99, 10.00); + put(19999.99, 9.30); + put(24999.99, 8.30); + put(29999.99, 7.80); + }}; + + private double periodCount; + private double premium; + private double downPercent; + private int paymentsCount; + private double expectedValue; + + @Parameterized.Parameters + public static Collection paramsAndExpectedVal() { + return Arrays.asList(new Object[][]{ + {12, 1100, 12, 10, -77.014302}, + {12, 2450, 12, 10, -164.843936}, + {12, 875, 12, 10, -61.261377}, + {12, 3850, 12, 10, -242.903207}, + {12, 6575, 12, 10, -325.710132}, + {12, 1800, 12, 10, -124.888837}, + {12, 11000, 12, 10, -544.914289}, + {12, 1136.36, 12, 10, -79.082437}, + }); + } + + @Test public void testCumipmt() throws Exception { + double financed = premium * (1 - downPercent / 100); + + double ratePercent = MAX_RATE_PERCENT; + for (Map.Entry item : RATE_PERCENT_MAPPING.entrySet()) { + if (ratePercent > item.getValue() && financed > item.getKey()) { + ratePercent = item.getValue(); + } + } + + double c1 = cumipmt(ratePercent / periodCount / 100, paymentsCount, financed, 1, paymentsCount, true); + assertEquals(expectedValue, c1, PRECISSION); + } +} diff --git a/src/test/java/org/cobbzilla/util/reflect/ReflectionUtilTest.java b/src/test/java/org/cobbzilla/util/reflect/ReflectionUtilTest.java new file mode 100644 index 0000000..813835f --- /dev/null +++ b/src/test/java/org/cobbzilla/util/reflect/ReflectionUtilTest.java @@ -0,0 +1,60 @@ +package org.cobbzilla.util.reflect; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; +import org.junit.Test; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.now; +import static org.junit.Assert.*; + +public class ReflectionUtilTest { + + @AllArgsConstructor + public static class Dummy { + @Getter @Setter public Long id; + + @Getter public String name; + + public void setName (String name) { + this.name = name; + } + public void setName (Dummy something) { + die("should not get called!"); + } + } + + private static final String ID = "id"; + public static final String NAME = "name"; + + @Test public void testGetSet () throws Exception { + + Long testValue = now(); + Dummy dummy = new Dummy(testValue, NAME); + assertEquals(ReflectionUtil.get(dummy, ID), testValue); + + ReflectionUtil.set(dummy, ID, null); + assertNull(ReflectionUtil.get(dummy, ID)); + + testValue += 10; + ReflectionUtil.set(dummy, ID, testValue); + assertEquals(ReflectionUtil.get(dummy, ID), testValue); + + ReflectionUtil.setNull(dummy, ID, Long.class); + assertNull(ReflectionUtil.get(dummy, ID)); + + ReflectionUtil.set(dummy, NAME, "a value"); + assertEquals(ReflectionUtil.get(dummy, NAME), "a value"); + + try { + ReflectionUtil.set(dummy, NAME, null); + fail("should not have been able to set name field to null"); + } catch (Exception expected) {} + + ReflectionUtil.setNull(dummy, NAME, String.class); + assertNull(ReflectionUtil.get(dummy, NAME)); + + + } +} diff --git a/src/test/java/org/cobbzilla/util/security/CryptoUtilTest.java b/src/test/java/org/cobbzilla/util/security/CryptoUtilTest.java new file mode 100644 index 0000000..a89512a --- /dev/null +++ b/src/test/java/org/cobbzilla/util/security/CryptoUtilTest.java @@ -0,0 +1,77 @@ +package org.cobbzilla.util.security; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.RandomUtils; +import org.cobbzilla.util.io.FileUtil; +import org.junit.Test; + +import java.io.*; + +import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.system.CommandShell.execScript; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class CryptoUtilTest { + + @Test public void testStringEncryption () throws Exception { + final String plaintext = randomAlphanumeric(50+RandomUtils.nextInt(100, 500)); + final String key = randomAlphanumeric(20); + final String ciphertext = CryptoUtil.string_encrypt(plaintext, key); + assertEquals(plaintext, CryptoUtil.string_decrypt(ciphertext, key)); + } + + @Test public void testRsa () throws Exception { + final RsaKeyPair pair1 = RsaKeyPair.newRsaKeyPair(); + final RsaKeyPair pair2 = RsaKeyPair.newRsaKeyPair(); + + final String p1data = randomAlphanumeric(1000); + final RsaMessage p1enc = pair1.encrypt(p1data, pair2); + + final String decrypted = pair2.decrypt(p1enc, pair1.setPrivateKey(null)); + assertEquals(decrypted, p1data); + + final String sshKey = pair2.getSshPublicKey(); + assertNotNull(sshKey); + } + + @Test public void testStream () throws Exception { + final String password = randomAlphanumeric(30); + final byte[] salt = RandomUtils.nextBytes(128); + final String aad = randomAlphanumeric(400); + + final CryptStream stream = new CryptStream(password); + final File temp = FileUtil.temp(".tmp"); + + // write a huge file to disk, approx 80MB + try (OutputStream out = new BufferedOutputStream(new FileOutputStream(temp))) { + for (int i=0; i<100000; i++) { + out.write(RandomUtils.nextBytes(8192)); + } + } + + // encrypt the file, save in another file + // if we stream correctly, we should not run out of memory + final File enc = new File(temp.getParentFile(), temp.getName()+".enc"); + try (InputStream in = stream.wrapWrite(new BufferedInputStream(new FileInputStream(temp)), salt, aad)) { + try (OutputStream out = new BufferedOutputStream(new FileOutputStream(enc))) { + IOUtils.copyLarge(in, out); + } + } + + // decrypt the file + // if we stream correctly, we should not run out of memory + final File dec = new File(temp.getParentFile(), temp.getName()+".dec"); + try (InputStream in = stream.wrapRead(new BufferedInputStream(new FileInputStream(enc)), salt, aad)) { + try (OutputStream out = new BufferedOutputStream(new FileOutputStream(dec))) { + IOUtils.copyLarge(in, out); + } + } + + // assert files are the same + final String result = execScript("diff "+abs(dec)+" "+abs(temp)); + assertEquals("expected no diff", "", result); + } + +} diff --git a/src/test/java/org/cobbzilla/util/string/StringUtilTest.java b/src/test/java/org/cobbzilla/util/string/StringUtilTest.java new file mode 100644 index 0000000..3265c41 --- /dev/null +++ b/src/test/java/org/cobbzilla/util/string/StringUtilTest.java @@ -0,0 +1,31 @@ +package org.cobbzilla.util.string; + +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class StringUtilTest { + + public static final String[][] TESTS = new String[][] { + {"simple", "simple"}, + {"two terms", "two", "terms"}, + {"multiple spaces", "multiple", "spaces"}, + {"first \"something in quotes\" last", "first", "something in quotes", "last"}, + {"first \"something in quotes with spaces\" last", "first", "something in quotes with spaces", "last"}, + {"first e:\"exact phrase\" last", "first", "e:", "exact phrase", "last"}, + {"R:first \"exact phrase\" last", "R:first", "exact phrase", "last"} + }; + + @Test public void testSplitIntoTerms () throws Exception { + for (String[] test : TESTS) { + final List terms = StringUtil.splitIntoTerms(test[0]); + assertEquals("wrong # terms: "+test[0], test.length-1, terms.size()); + for (int i=1; i foundPercentages = new ArrayList<>(5); + + @Override protected void processLine(String line) throws IOException { + if (line.startsWith("Process is")) { + final Matcher matcher = PCT_COMPLETE_PATTERN.matcher(line); + if (matcher.find()) { + foundPercentages.add(Integer.valueOf(matcher.group(1))); + } + } + } + + } + + private class TestProgressCallback implements CommandProgressCallback { + + public Set patternsFound = new HashSet<>(); + public int count = 0; + + @Override public void updateProgress(CommandProgressMarker marker) { + patternsFound.add(marker.getPattern().toString()); + count++; + } + } +} diff --git a/src/test/java/org/cobbzilla/util/time/ImprovedTimezoneTest.java b/src/test/java/org/cobbzilla/util/time/ImprovedTimezoneTest.java new file mode 100644 index 0000000..57f1234 --- /dev/null +++ b/src/test/java/org/cobbzilla/util/time/ImprovedTimezoneTest.java @@ -0,0 +1,26 @@ +package org.cobbzilla.util.time; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; + +import java.io.File; + +import static org.junit.Assert.assertTrue; + +@Slf4j +public class ImprovedTimezoneTest { + + @Test + public void validate () { + + if (!System.getProperty("os.name").toLowerCase().contains("linux")) { + log.warn("validate: skipping, this only runs on Linux"); + } + + // validates that all Java timezones properly map to Linux timezones, and those timezones exist + for (ImprovedTimezone tz : ImprovedTimezone.getTimeZones()) { + assertTrue(new File("/usr/share/zoneinfo/"+tz.getLinuxName()).exists()); + } + } + +} diff --git a/src/test/java/org/cobbzilla/util/time/JavaTimezoneTest.java b/src/test/java/org/cobbzilla/util/time/JavaTimezoneTest.java new file mode 100644 index 0000000..bd78787 --- /dev/null +++ b/src/test/java/org/cobbzilla/util/time/JavaTimezoneTest.java @@ -0,0 +1,32 @@ +package org.cobbzilla.util.time; + +import org.junit.Test; + +import java.util.Collection; +import java.util.List; + +import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.time.UnicodeTimezone.getAllUnicodeTimezones; +import static org.junit.Assert.assertNotNull; + +public class JavaTimezoneTest { + + @Test public void ensureJavaTimezoneExistsForEveryUnicodeTimezone () throws Exception { + final Collection allUnicode = getAllUnicodeTimezones(); + for (UnicodeTimezone utz : allUnicode) { + if (utz.deprecated()) continue; // skip deprecated time zones, we may not have a mapping and that is OK + final JavaTimezone javaForUnicode = JavaTimezone.fromUnicode(utz); + assertNotNull("no JavaTimezone found for utz="+json(utz, COMPACT_MAPPER), javaForUnicode); + } + } + + @Test public void ensureJavaTimezoneExistsForEveryLinuxTimezone () throws Exception { + final List allLinux = LinuxTimezone.getAllLinuxTimezones(); + for (LinuxTimezone linuxTimezone : allLinux) { + final JavaTimezone javaForLinux = JavaTimezone.fromLinux(linuxTimezone); + assertNotNull("no JavaTimezone found for linuxTimezone="+linuxTimezone.getName(), javaForLinux); + } + } + +} diff --git a/src/test/java/org/cobbzilla/util/time/LinuxTimezoneTest.java b/src/test/java/org/cobbzilla/util/time/LinuxTimezoneTest.java new file mode 100644 index 0000000..2c2c082 --- /dev/null +++ b/src/test/java/org/cobbzilla/util/time/LinuxTimezoneTest.java @@ -0,0 +1,22 @@ +package org.cobbzilla.util.time; + +import org.junit.Test; + +import java.util.Collection; + +import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.time.UnicodeTimezone.getAllUnicodeTimezones; +import static org.junit.Assert.assertNotNull; + +public class LinuxTimezoneTest { + + @Test public void ensureLinuxTimezoneExistsForEveryUnicodeTimezone () throws Exception { + final Collection allUnicode = getAllUnicodeTimezones(); + for (UnicodeTimezone utz : allUnicode) { + if (utz.deprecated()) continue; // skip deprecated time zones, we may not have a mapping and that is OK + final LinuxTimezone linuxForUnicode = LinuxTimezone.fromUnicode(utz); + assertNotNull("no LinuxTimezone found for utz="+json(utz, COMPACT_MAPPER), linuxForUnicode); + } + } +} diff --git a/src/test/resources/org/cobbzilla/util/handlebars/alt_test.txt b/src/test/resources/org/cobbzilla/util/handlebars/alt_test.txt new file mode 100644 index 0000000..b3d0c23 --- /dev/null +++ b/src/test/resources/org/cobbzilla/util/handlebars/alt_test.txt @@ -0,0 +1,11 @@ +congrats: + common: | + "# Congratulations! #" + "# Your TEST_NAME server is running. #" + "# Local DNS resolver {{ local_service_ip }}{{ ', ' + local_service_ipv6 if ipv6_support else '' }} #" + p12_pass: | + "# The p12 and SSH keys password for new users is {{ p12_export_password }} #" + ca_key_pass: | + "# The CA key password is {{ CA_password|default(omit) }} #" + ssh_access: | + "# Shell access: ssh -i {{ ansible_ssh_private_key_file|default(omit) }} {{ ansible_ssh_user|default(omit) }}@{{ ansible_ssh_host|default(omit) }} #" diff --git a/src/test/resources/org/cobbzilla/util/handlebars/alt_test.txt.hbs b/src/test/resources/org/cobbzilla/util/handlebars/alt_test.txt.hbs new file mode 100644 index 0000000..5b5cd1f --- /dev/null +++ b/src/test/resources/org/cobbzilla/util/handlebars/alt_test.txt.hbs @@ -0,0 +1,11 @@ +congrats: + common: | + "# Congratulations! #" + "# Your <> server is running. #" + "# Local DNS resolver {{ local_service_ip }}{{ ', ' + local_service_ipv6 if ipv6_support else '' }} #" + p12_pass: | + "# The p12 and SSH keys password for new users is {{ p12_export_password }} #" + ca_key_pass: | + "# The CA key password is {{ CA_password|default(omit) }} #" + ssh_access: | + "# Shell access: ssh -i {{ ansible_ssh_private_key_file|default(omit) }} {{ ansible_ssh_user|default(omit) }}@{{ ansible_ssh_host|default(omit) }} #" diff --git a/src/test/resources/org/cobbzilla/util/io/test.tar.gz b/src/test/resources/org/cobbzilla/util/io/test.tar.gz new file mode 100644 index 0000000..025f164 Binary files /dev/null and b/src/test/resources/org/cobbzilla/util/io/test.tar.gz differ diff --git a/src/test/resources/org/cobbzilla/util/json/merge/test1_expected.json b/src/test/resources/org/cobbzilla/util/json/merge/test1_expected.json new file mode 100644 index 0000000..ace72d4 --- /dev/null +++ b/src/test/resources/org/cobbzilla/util/json/merge/test1_expected.json @@ -0,0 +1,8 @@ +{ + "name": "something", + "nested": { + "field": "foo", + "field2": "bar" + }, + "other": "baz" +} \ No newline at end of file diff --git a/src/test/resources/org/cobbzilla/util/json/merge/test1_orig.json b/src/test/resources/org/cobbzilla/util/json/merge/test1_orig.json new file mode 100644 index 0000000..8c37b82 --- /dev/null +++ b/src/test/resources/org/cobbzilla/util/json/merge/test1_orig.json @@ -0,0 +1,6 @@ +{ + "name": "something", + "nested": { + "field": "foo" + } +} \ No newline at end of file diff --git a/src/test/resources/org/cobbzilla/util/json/merge/test1_request.json b/src/test/resources/org/cobbzilla/util/json/merge/test1_request.json new file mode 100644 index 0000000..62483c7 --- /dev/null +++ b/src/test/resources/org/cobbzilla/util/json/merge/test1_request.json @@ -0,0 +1,7 @@ +{ + "name": "something", + "nested": { + "field2": "bar" + }, + "other": "baz" +} \ No newline at end of file diff --git a/src/test/resources/org/cobbzilla/util/json/test.json b/src/test/resources/org/cobbzilla/util/json/test.json new file mode 100644 index 0000000..e4129ab --- /dev/null +++ b/src/test/resources/org/cobbzilla/util/json/test.json @@ -0,0 +1,20 @@ +{ + "id": "test", + "thing": { + "field1": [ "item1", "item2", "item3" ], + "field2": { + "subfieldA": "foo", + "subB": "bar" + } + }, + "another_thing": { + "field1": "baz", + "fieldZ": { + "nested": { + "deeper": { + "quux": "blah" + } + } + } + } +} \ No newline at end of file diff --git a/src/test/resources/org/cobbzilla/util/system/test.sh b/src/test/resources/org/cobbzilla/util/system/test.sh new file mode 100755 index 0000000..7371f99 --- /dev/null +++ b/src/test/resources/org/cobbzilla/util/system/test.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Simulates a long-running process for CommandShellTest +# Periodically prints out certain lines that match regexes for measuring command progress + +i=1 +limit=500 +while [ ${i} -le ${limit} ] ; do + rand=$(od -An -d /dev/urandom | head -n 1 | awk '{print $1}') + num_rand=$(expr ${rand} % 3) + for j in {0..${num_rand}} ; do + echo "some random line: ${rand}" + done + if [ $(expr ${i} % 100) -eq 0 ] ; then + pct=$(expr 100 \* ${i} / ${limit}) + echo -n "Process is ${pct}% complete: " + case ${pct} in + 20) echo "marker::cat" ;; + 40) echo "marker::dog" ;; + 60) echo "marker::bird" ;; + 80) echo "marker::fish" ;; + 100) echo "marker::hedgehog" ;; + esac + fi + i=$(expr ${i} + 1) +done \ No newline at end of file