add more network/hostname utilities

src/main/java/org/cobbzilla/util/http/HttpUtil.java

@@ -73,13 +73,22 @@ public class HttpUtil {
 
     public static InputStream get (String urlString) throws IOException { return get(urlString, null); }
 
+    public static InputStream get (String urlString, long connectTimeout) throws IOException {
+        return get(urlString, null, null, connectTimeout);
+    }
+
     public static InputStream get (String urlString, Map<String, String> headers) throws IOException {
         return get(urlString, headers, null);
     }
 
     public static InputStream get (String urlString, Map<String, String> headers, Map<String, String> headers2) throws IOException {
+        return get(urlString, headers, headers2, null);
+    }
+
+    public static InputStream get (String urlString, Map<String, String> headers, Map<String, String> headers2, Long connectTimeout) throws IOException {
         final URL url = new URL(urlString);
         final HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
+        if (connectTimeout != null) urlConnection.setConnectTimeout(connectTimeout.intValue());
         if (headers != null) addHeaders(urlConnection, headers);
         if (headers2 != null) addHeaders(urlConnection, headers2);
         return urlConnection.getInputStream();
@@ -198,6 +207,13 @@ public class HttpUtil {
         return out.toString();
     }
 
+    public static String url2string (String url, long connectTimeout) throws IOException {
+        @Cleanup final InputStream in = get(url, connectTimeout);
+        final ByteArrayOutputStream out = new ByteArrayOutputStream();
+        IOUtils.copy(in, out);
+        return out.toString();
+    }
+
     public static HttpResponseBean getResponse(HttpRequestBean requestBean) throws IOException {
         final HttpClientBuilder clientBuilder = requestBean.initClientBuilder(HttpClients.custom());
         @Cleanup final CloseableHttpClient client = clientBuilder.build();

src/main/java/org/cobbzilla/util/network/NetworkUtil.java

@@ -4,17 +4,21 @@ import com.sun.jna.Platform;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.util.collection.ExpirationMap;
+import org.cobbzilla.util.io.FileUtil;
 
+import java.io.File;
+import java.io.IOException;
 import java.net.InetAddress;
 import java.net.NetworkInterface;
-import java.util.Enumeration;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
+import java.util.stream.Collectors;
 
 import static org.cobbzilla.util.daemon.ZillaRuntime.die;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.http.HttpSchemes.SCHEME_HTTPS;
+import static org.cobbzilla.util.http.HttpSchemes.isHttpOrHttps;
 import static org.cobbzilla.util.http.HttpUtil.url2string;
+import static org.cobbzilla.util.http.URIUtil.toUri;
 import static org.cobbzilla.util.string.ValidationRegexes.IPv4_PATTERN;
 
 @Slf4j
@@ -212,4 +216,20 @@ public class NetworkUtil {
             return false;
         }
     }
+
+    public static Set<String> toHostSet(File file) throws IOException {
+        return toHostSet(FileUtil.toStringList(file));
+    }
+
+    public static Set<String> toHostSet(Collection<String> vals) {
+        return vals.stream()
+                .map(NetworkUtil::normalizeHost)
+                .collect(Collectors.toSet());
+    }
+
+    public static String normalizeHost(String s) {
+        return isHttpOrHttps(s)
+                ? toUri(s).getHost()
+                : toUri(SCHEME_HTTPS + s).getHost();
+    }
 }