diff --git a/src/main/java/org/cobbzilla/util/http/HttpUtil.java b/src/main/java/org/cobbzilla/util/http/HttpUtil.java index 24830fb..667c85b 100644 --- a/src/main/java/org/cobbzilla/util/http/HttpUtil.java +++ b/src/main/java/org/cobbzilla/util/http/HttpUtil.java @@ -73,13 +73,22 @@ public class HttpUtil { public static InputStream get (String urlString) throws IOException { return get(urlString, null); } + public static InputStream get (String urlString, long connectTimeout) throws IOException { + return get(urlString, null, null, connectTimeout); + } + public static InputStream get (String urlString, Map headers) throws IOException { return get(urlString, headers, null); } public static InputStream get (String urlString, Map headers, Map headers2) throws IOException { + return get(urlString, headers, headers2, null); + } + + public static InputStream get (String urlString, Map headers, Map headers2, Long connectTimeout) throws IOException { final URL url = new URL(urlString); final HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); + if (connectTimeout != null) urlConnection.setConnectTimeout(connectTimeout.intValue()); if (headers != null) addHeaders(urlConnection, headers); if (headers2 != null) addHeaders(urlConnection, headers2); return urlConnection.getInputStream(); @@ -198,6 +207,13 @@ public class HttpUtil { return out.toString(); } + public static String url2string (String url, long connectTimeout) throws IOException { + @Cleanup final InputStream in = get(url, connectTimeout); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOUtils.copy(in, out); + return out.toString(); + } + public static HttpResponseBean getResponse(HttpRequestBean requestBean) throws IOException { final HttpClientBuilder clientBuilder = requestBean.initClientBuilder(HttpClients.custom()); @Cleanup final CloseableHttpClient client = clientBuilder.build(); diff --git a/src/main/java/org/cobbzilla/util/network/NetworkUtil.java b/src/main/java/org/cobbzilla/util/network/NetworkUtil.java index 0e2feb6..ccc7eba 100644 --- a/src/main/java/org/cobbzilla/util/network/NetworkUtil.java +++ b/src/main/java/org/cobbzilla/util/network/NetworkUtil.java @@ -4,17 +4,21 @@ import com.sun.jna.Platform; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.util.collection.ExpirationMap; +import org.cobbzilla.util.io.FileUtil; +import java.io.File; +import java.io.IOException; import java.net.InetAddress; import java.net.NetworkInterface; -import java.util.Enumeration; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; +import java.util.stream.Collectors; import static org.cobbzilla.util.daemon.ZillaRuntime.die; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.http.HttpSchemes.SCHEME_HTTPS; +import static org.cobbzilla.util.http.HttpSchemes.isHttpOrHttps; import static org.cobbzilla.util.http.HttpUtil.url2string; +import static org.cobbzilla.util.http.URIUtil.toUri; import static org.cobbzilla.util.string.ValidationRegexes.IPv4_PATTERN; @Slf4j @@ -212,4 +216,20 @@ public class NetworkUtil { return false; } } + + public static Set toHostSet(File file) throws IOException { + return toHostSet(FileUtil.toStringList(file)); + } + + public static Set toHostSet(Collection vals) { + return vals.stream() + .map(NetworkUtil::normalizeHost) + .collect(Collectors.toSet()); + } + + public static String normalizeHost(String s) { + return isHttpOrHttps(s) + ? toUri(s).getHost() + : toUri(SCHEME_HTTPS + s).getHost(); + } }