improve storage testing

bubble-server/src/main/java/bubble/cloud/storage/StorageServiceDriver.java

@@ -18,7 +18,9 @@ import java.io.InputStream;
 
 import static bubble.ApiConstants.ROOT_NETWORK_UUID;
 import static java.util.UUID.randomUUID;
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
 import static org.cobbzilla.util.daemon.ZillaRuntime.*;
+import static org.cobbzilla.util.io.StreamUtil.toStringOrDie;
 import static org.cobbzilla.util.system.Sleep.sleep;
 
 public interface StorageServiceDriver extends CloudServiceDriver {
@@ -116,18 +118,17 @@ public interface StorageServiceDriver extends CloudServiceDriver {
     }
 
     @Override default boolean test () {
-        try {
-            if (!write(getTestNodeId(), getTestKey(), getTestBytes())) return false;
-            if (!delete(getTestNodeId(), getTestKey())) return false;
-        } catch (Exception e) {
-            return false;
-        }
-        return true;
+        final String node = getTestNodeId();
+        final String key = getTestKey();
+        final String data = getTestData();
+        return write(node, key, data.getBytes())
+                && toStringOrDie(read(node, key)).equals(data)
+                && delete(node, key);
     }
 
     default String getTestNodeId () { return ROOT_NETWORK_UUID; }
-    default String getTestKey () { return "_test"; }
-    default byte[] getTestBytes() { return "test".getBytes(); }
+    default String getTestKey    () { return "driver_test_key_" +randomAlphanumeric(30); }
+    default String getTestData   () { return "driver_test_data_"+randomAlphanumeric(100); }
 
     boolean _write(String fromNode, String key, InputStream data, StorageMetadata metadata, String requestId) throws IOException;
 
@@ -143,7 +144,7 @@ public interface StorageServiceDriver extends CloudServiceDriver {
                 sleep(DEFAULT_RETRY_BACKOFF.apply(i), "waiting to retry _write");
             }
         }
-        return die(lastEx == null ? "write error, no lastEx" : "write: "+lastEx);
+        return die(lastEx == null ? "write error, no lastEx" : "write: "+shortError(lastEx));
     }
 
     default boolean write(String fromNode, String key, InputStream data, StorageMetadata metadata) {
@@ -182,7 +183,7 @@ public interface StorageServiceDriver extends CloudServiceDriver {
 
     boolean canWrite(String fromNode, String toNode, String key);
 
-    boolean delete(String fromNode, String uri) throws IOException;
+    boolean delete(String fromNode, String uri);
     boolean deleteNetwork(String networkUuid) throws IOException;
 
     boolean rekey(String fromNode, CloudService newCloud) throws IOException;

bubble-server/src/main/java/bubble/cloud/storage/delegate/DelegatedStorageDriver.java

@@ -74,7 +74,7 @@ public class DelegatedStorageDriver extends DelegatedStorageDriverBase {
         return to != null && to.getNetwork().equals(configuration.getThisNetwork().getUuid());
     }
 
-    @Override public boolean delete(String fromNode, String key) throws IOException {
+    @Override public boolean delete(String fromNode, String key) {
         return booleanRequest(key, storage_driver_delete);
     }
 

bubble-server/src/main/java/bubble/cloud/storage/local/LocalStorageDriver.java

@@ -155,10 +155,14 @@ public class LocalStorageDriver extends CloudServiceDriverBase<LocalStorageConfi
         return from != null && from.getNetwork().equals(configuration.getThisNetwork().getUuid());
     }
 
-    @Override public boolean delete(String fromNode, String uri) throws IOException {
+    @Override public boolean delete(String fromNode, String uri) {
         final BubbleNode from = getFromNode(fromNode);
         final File file = keyFile(from, uri);
-        FileUtils.forceDelete(file);
+        try {
+            FileUtils.forceDelete(file);
+        } catch (IOException e) {
+            return die("delete: forceDelete("+abs(file)+") failed: "+shortError(e));
+        }
         return true;
     }
 

bubble-server/src/main/java/bubble/cloud/storage/s3/S3StorageDriver.java

@@ -156,7 +156,7 @@ public class S3StorageDriver extends StorageServiceDriverBase<S3StorageConfig> {
                 final Map<String, String> userMetadata = objectMetadata.getUserMetadata();
                 final StorageMetadata remoteMeta = StorageMetadata.fromMap(userMetadata);
                 if (remoteMeta.sameSha(metadata.getSha256()) && !metadata.isForceWrite()) {
-                    log.info("_write: sha256 matches, not writing (but returning true): for key="+key);
+                    log.info("writeStorage: sha256 matches, not writing (but returning true): for key="+key);
                     return true;
                 } else {
                     userMetadata.put(META_MTIME, ""+now());
@@ -186,11 +186,11 @@ public class S3StorageDriver extends StorageServiceDriverBase<S3StorageConfig> {
             return true;
 
         } catch (Exception e) {
-            throw new IOException("_write: "+e);
+            throw new IOException("writeStorage: "+e);
         }
     }
 
-    @Override public boolean delete(String fromNode, String uri) throws IOException {
+    @Override public boolean delete(String fromNode, String uri) {
         final BubbleNode from = getFromNode(fromNode);
         final AmazonS3 s3client = getS3client();
         final String key = s3path(from, uri);

bubble-server/src/main/java/bubble/service/boot/StandardSelfNodeService.java

@@ -164,7 +164,7 @@ public class StandardSelfNodeService implements SelfNodeService {
                 log.debug("initThisNode: wasRestored=false, just returning self: "+initSelf.id());
                 return initSelf;
             } else if (self == NULL_NODE) {
-                log.warn("getThisNode: initThisNode returned NULL_NODE");
+                if (!nullWarningPrinted.check()) log.warn("getThisNode: initThisNode returned NULL_NODE");
                 return null;
             } else {
                 log.debug("getThisNode: thisNode already set, returning: "+self.id());