improve field type detection

wizard-server/src/main/java/org/cobbzilla/wizard/dao/SqlDefaultSearchField.java → wizard-common/src/main/java/org/cobbzilla/wizard/model/SqlDefaultSearchField.java

@@ -1,10 +1,12 @@
-package org.cobbzilla.wizard.dao;
+package org.cobbzilla.wizard.model;
 
 import lombok.EqualsAndHashCode;
+import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.wizard.model.entityconfig.EntityFieldType;
 import org.cobbzilla.wizard.model.entityconfig.annotations.ECField;
 import org.cobbzilla.wizard.model.entityconfig.annotations.ECSearchable;
 import org.cobbzilla.wizard.model.search.SearchBound;
+import org.cobbzilla.wizard.model.search.SearchBoundBuilder;
 import org.cobbzilla.wizard.model.search.SearchField;
 import org.cobbzilla.wizard.model.search.SearchFieldType;
 
@@ -20,7 +22,7 @@ import static org.cobbzilla.util.string.StringUtil.camelCaseToSnakeCase;
 import static org.cobbzilla.wizard.model.entityconfig.EntityFieldType.*;
 import static org.cobbzilla.wizard.model.search.SearchBoundComparison.*;
 
-@EqualsAndHashCode
+@EqualsAndHashCode @Slf4j
 public class SqlDefaultSearchField implements SearchField {
 
     private final Field f;
@@ -30,6 +32,10 @@ public class SqlDefaultSearchField implements SearchField {
     private final List<Object> params;
     private final String locale;
 
+    public SqlDefaultSearchField(Field f, ECSearchable search, String bound) {
+        this(f, search, bound, null, null, null);
+    }
+
     public SqlDefaultSearchField(Field f, ECSearchable search, String bound, String value, List<Object> params, String locale) {
         this.f = f;
         this.search = search;
@@ -42,7 +48,7 @@ public class SqlDefaultSearchField implements SearchField {
     @Override public String name() { return camelCaseToSnakeCase(f.getName()); }
 
     @Override public SearchBound[] getBounds() {
-        List<SearchBound> bounds = new ArrayList<>();
+        final List<SearchBound> bounds = new ArrayList<>();
         EntityFieldType fieldType = search.type();
         if (fieldType == none_set) {
             if (!empty(search.bounds())) {
@@ -54,7 +60,7 @@ public class SqlDefaultSearchField implements SearchField {
                 }
             } else {
                 final ECField ecField = f.getAnnotation(ECField.class);
-                fieldType = ecField != null ? ecField.type() : null;
+                fieldType = ecField != null ? ecField.type() : guessFieldType(f);
             }
         }
         if (fieldType != null) {
@@ -65,20 +71,25 @@ public class SqlDefaultSearchField implements SearchField {
                 case expiration_time:
                     bounds.addAll(asList(SearchField.bindTime(name())));
                     break;
+                case integer: case money_integer:
+                    bounds.addAll(asList(SearchField.bindInteger(name())));
+                    break;
+                case decimal: case money_decimal:
+                    bounds.addAll(asList(SearchField.bindDecimal(name())));
+                    break;
                 case flag:
-                    bounds.add(eq.bind(name(), SearchFieldType.integer));
+                    bounds.add(eq.bind(name(), SearchFieldType.flag));
                     break;
-                case string:
-                    bounds.add(eq.bind(name(), SearchFieldType.string));
+                case string: case email: case time_zone: case locale:
+                case ip4: case ip6: case http_url:
+                case us_phone: case us_state: case us_zip:
+                    bounds.addAll(asList(SearchField.bindString(f, name())));
                     break;
             }
             if (isNullable(f)) {
                 bounds.add(is_null.bind(name()));
                 bounds.add(not_null.bind(name()));
             }
-            if (safeColumnLength(f) > 500) {
-                bounds.add(like.bind(name()));
-            }
         }
         if (empty(bounds)) return die("getBounds: no bounds defined for: "+ bound);
         return bounds.toArray(SearchBound[]::new);

wizard-common/src/main/java/org/cobbzilla/wizard/model/entityconfig/EntityFieldType.java

@@ -2,6 +2,7 @@ package org.cobbzilla.wizard.model.entityconfig;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.wizard.model.entityconfig.validation.*;
 import org.cobbzilla.wizard.validation.ValidationResult;
 import org.cobbzilla.wizard.validation.Validator;
@@ -12,7 +13,7 @@ import java.util.Locale;
 
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 
-@AllArgsConstructor
+@AllArgsConstructor @Slf4j
 public enum EntityFieldType {
 
     /** it holds a place where nothing was set */
@@ -144,6 +145,39 @@ public enum EntityFieldType {
         return column == null ? null : column.length();
     }
 
+    public static EntityFieldType guessFieldType(Field f) {
+        switch (f.getType().getName()) {
+            case "boolean":
+            case "java.lang.Boolean":
+                return flag;
+            case "long":
+            case "java.lang.Long":
+                if (f.getName().equals("ctime") || f.getName().equals("mtime")) return epoch_time;
+            case "byte":
+            case "short":
+            case "int":
+            case "java.lang.Byte":
+            case "java.lang.Short":
+            case "java.lang.Integer":
+            case "java.math.BigInteger":
+                return integer;
+            case "char":
+            case "java.lang.Character":
+            case "java.lang.String":
+                return string;
+            case "float":
+            case "double":
+            case "java.lang.Float":
+            case "java.lang.Double":
+            case "java.math.BigDecimal":
+                return decimal;
+            default:
+                if (f.getType().isEnum()) return string;
+                log.warn("guessFieldType: unrecognized type ("+f.getType().getName()+") for field: "+f.getName());
+                return null;
+        }
+    }
+
     public Object toObject(Locale locale, String value) {
         return fieldValidator == null ? value : fieldValidator.toObject(locale, value);
     }

wizard-server/src/main/java/org/cobbzilla/wizard/dao/SearchBoundBuilder.java → wizard-common/src/main/java/org/cobbzilla/wizard/model/search/SearchBoundBuilder.java

@@ -1,4 +1,4 @@
-package org.cobbzilla.wizard.dao;
+package org.cobbzilla.wizard.model.search;
 
 import org.cobbzilla.wizard.model.search.SearchBound;
 

wizard-common/src/main/java/org/cobbzilla/wizard/model/search/SearchField.java

@@ -2,6 +2,7 @@ package org.cobbzilla.wizard.model.search;
 
 import org.cobbzilla.wizard.validation.SimpleViolationException;
 
+import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -22,6 +23,33 @@ public interface SearchField {
     }
 
     static SearchBound[] bindTime(String name) { return new SearchBound[] { during.bind(name), after.bind(name), before.bind(name) }; }
+    static SearchBound[] bindInteger(String name) {
+        return new SearchBound[] {
+                eq.bind(name, SearchFieldType.integer),
+                lt.bind(name, SearchFieldType.integer),
+                le.bind(name, SearchFieldType.integer),
+                gt.bind(name, SearchFieldType.integer),
+                ge.bind(name, SearchFieldType.integer),
+                ne.bind(name, SearchFieldType.integer)
+        };
+    }
+    static SearchBound[] bindDecimal(String name) {
+        return new SearchBound[] {
+                eq.bind(name, SearchFieldType.decimal),
+                lt.bind(name, SearchFieldType.decimal),
+                le.bind(name, SearchFieldType.decimal),
+                gt.bind(name, SearchFieldType.decimal),
+                ge.bind(name, SearchFieldType.decimal),
+                ne.bind(name, SearchFieldType.decimal)
+        };
+    }
+    static SearchBound[] bindString(Field f, String name) {
+        return new SearchBound[] {
+                eq.bind(name, SearchFieldType.string),
+                ne.bind(name, SearchFieldType.string),
+                like.bind(name, SearchFieldType.string)
+        };
+    }
     static SearchBound[] bindNullable(String name) { return new SearchBound[] { eq.bind(name), is_null.bind(name), not_null.bind(name) }; }
 
     String name();

wizard-common/src/main/java/org/cobbzilla/wizard/model/search/SqlViewField.java

@@ -28,6 +28,8 @@ public class SqlViewField {
     @JsonIgnore @Getter @Setter private SqlViewFieldSetter setter;
     public boolean hasSetter () { return setter != null; }
 
+    @Getter @Setter private SearchBound[] bounds;
+
     public SqlViewField(String name) {
         this.name = name;
         this.property = snakeCaseToCamelCase(name);

wizard-common/src/main/java/org/cobbzilla/wizard/model/search/SqlViewSearchResult.java

@@ -1,10 +1,30 @@
 package org.cobbzilla.wizard.model.search;
 
+import org.cobbzilla.util.collection.ExpirationMap;
 import org.cobbzilla.wizard.model.HasRelatedEntities;
 import org.cobbzilla.wizard.model.RelatedEntities;
+import org.cobbzilla.wizard.model.SqlDefaultSearchField;
+import org.cobbzilla.wizard.model.entityconfig.annotations.ECSearchable;
+
+import java.lang.reflect.Field;
+import java.util.Map;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.die;
+import static org.cobbzilla.util.reflect.ReflectionUtil.fieldsWithAnnotation;
 
 public interface SqlViewSearchResult extends HasRelatedEntities {
 
     RelatedEntities getRelated();
 
+    Map<String, SearchField> boundCache = new ExpirationMap<>();
+
+    default SearchField searchField(String bound) {
+        for (Field f : fieldsWithAnnotation(getClass(), ECSearchable.class)) {
+            if (!f.getName().equalsIgnoreCase(bound)) continue;
+            final ECSearchable search = f.getAnnotation(ECSearchable.class);
+            return boundCache.computeIfAbsent(bound, k -> new SqlDefaultSearchField(f, search, bound));
+        }
+        return die("genericBound: no bound defined for: "+bound);
+    }
+
 }

wizard-server/src/main/java/org/cobbzilla/wizard/dao/SearchResults.java

@@ -2,7 +2,6 @@ package org.cobbzilla.wizard.dao;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.databind.JavaType;
-import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
@@ -10,6 +9,7 @@ import lombok.experimental.Accessors;
 import org.cobbzilla.util.json.JsonUtil;
 import org.cobbzilla.wizard.filters.Scrubbable;
 import org.cobbzilla.wizard.filters.ScrubbableField;
+import org.cobbzilla.wizard.model.search.SearchQuery;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -19,12 +19,18 @@ import java.util.concurrent.ConcurrentHashMap;
 import static org.cobbzilla.util.daemon.ZillaRuntime.die;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 
-@NoArgsConstructor @AllArgsConstructor @Accessors(chain=true)
+@NoArgsConstructor @Accessors(chain=true)
 public class SearchResults<E> implements Scrubbable {
 
     public static final ScrubbableField[] SCRUBBABLE_FIELDS = new ScrubbableField[]{
             new ScrubbableField(SearchResults.class, "results.*", List.class)
     };
+
+    public SearchResults(List<E> results, int totalCount) {
+        this.results = results;
+        this.totalCount = totalCount;
+    }
+
     @Override public ScrubbableField[] fieldsToScrub() { return SCRUBBABLE_FIELDS; }
 
     private static Map<Class, JavaType> jsonTypeCache = new ConcurrentHashMap<>();
@@ -39,6 +45,7 @@ public class SearchResults<E> implements Scrubbable {
 
     @Getter @Setter private List<E> results = new ArrayList<>();
     @Getter @Setter private Integer totalCount;
+    @Getter @Setter private String nextPage;
 
     public String getResultType() { return empty(results) ? null : results.get(0).getClass().getName(); }
     public void setResultType (String val) {} // noop
@@ -65,4 +72,8 @@ public class SearchResults<E> implements Scrubbable {
         return this;
     }
 
+    public boolean hasNextPage(SearchQuery searchQuery) {
+        return getTotalCount() > searchQuery.getPageNumber() * searchQuery.getPageSize();
+    }
+
 }

wizard-server/src/main/java/org/cobbzilla/wizard/dao/SearchViewContext.java

@@ -2,20 +2,21 @@ package org.cobbzilla.wizard.dao;
 
 import com.github.jknack.handlebars.Handlebars;
 import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.util.handlebars.HandlebarsUtil;
 import org.cobbzilla.wizard.model.Identifiable;
 import org.cobbzilla.wizard.model.entityconfig.annotations.ECForeignKey;
 import org.cobbzilla.wizard.model.entityconfig.annotations.ECForeignKeySearchDepth;
 import org.cobbzilla.wizard.model.entityconfig.annotations.ECSearchDepth;
 import org.cobbzilla.wizard.model.entityconfig.annotations.ECSearchable;
-import org.cobbzilla.wizard.model.search.SqlViewField;
-import org.cobbzilla.wizard.model.search.SqlViewFieldSetter;
+import org.cobbzilla.wizard.model.search.*;
 
 import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.daemon.ZillaRuntime.hashOf;
@@ -28,6 +29,7 @@ import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.isEncryptedField;
 import static org.cobbzilla.wizard.model.entityconfig.annotations.ECForeignKeySearchDepth.*;
 import static org.cobbzilla.wizard.server.config.PgRestServerConfiguration.safeDbName;
 
+@Slf4j
 public class SearchViewContext {
 
     @Getter(lazy=true) private static final String defaultViewTemplate = stream2string(getPackagePath(AbstractDAO.class)+"/default_search_view.sql.hbs");
@@ -67,14 +69,21 @@ public class SearchViewContext {
 
     @Getter private final SqlViewField[] searchFields;
 
+    private static int longestFieldSet = 0;
+
     private SqlViewField[] initSearchFields() {
         final Map<String, SqlViewField> fields = new LinkedHashMap<>();
         final ECSearchDepth mainSearchDepth = clazz.getAnnotation(ECSearchDepth.class);
         final ECForeignKeySearchDepth mainDepth = mainSearchDepth == null ? inherit : mainSearchDepth.fkDepth();
         final Map<String, SqlViewField> finalizedFields = initFields(clazz, "", fields, mainDepth, mainDepth);
+        if (finalizedFields.size() > longestFieldSet) {
+            longestFieldSet = finalizedFields.size();
+        }
         return finalizedFields.values().toArray(new SqlViewField[0]);
     }
 
+    private static final Map<String, SearchBound[]> fieldBounds = new ConcurrentHashMap<>();
+
     private Map<String, SqlViewField> initFields(Class<? extends Identifiable> entityClass,
                                                  String prefix,
                                                  Map<String, SqlViewField> fields,
@@ -99,6 +108,16 @@ public class SearchViewContext {
                 final String entity = !empty(search.entity()) ? search.entity() : empty(prefix) ? null : prefix;
 
                 addColumn(viewFieldName, empty(prefix) ? entityTable : prefix, fieldName);
+
+                // calculate search field
+                final String sfKey = entityClass.getName() + "." + f.getName();
+                SearchBound[] bounds = null;
+                try {
+                    bounds = fieldBounds.computeIfAbsent(sfKey, k -> ((SqlViewSearchResult) instantiate(entityClass)).searchField(f.getName()).getBounds());
+                } catch (Exception e) {
+                    log.warn("initFields: error building SearchField for "+entityClass.getSimpleName()+"."+f.getName()+": "+e);
+                }
+
                 fields.putIfAbsent(viewFieldName, new SqlViewField(viewFieldName)
                         .setType(entityClass)
                         .fieldType(f.getType())
@@ -106,9 +125,11 @@ public class SearchViewContext {
                         .filter(search.filter())
                         .property(property)
                         .entity(entity)
-                        .setter(set));
+                        .setter(set)
+                        .setBounds(bounds));
 
                 if (fk != null) {
+                    if (!fk.cascade()) continue;
                     if (mainDepth == none) continue;
                     if (currentDepth == none) continue;
 

wizard-server/src/main/java/org/cobbzilla/wizard/dao/SqlViewSearchableDAO.java

@@ -4,11 +4,11 @@ import lombok.AllArgsConstructor;
 import lombok.Getter;
 import net.sf.cglib.proxy.Enhancer;
 import net.sf.cglib.proxy.InvocationHandler;
-import org.apache.commons.lang3.reflect.FieldUtils;
 import org.cobbzilla.util.collection.ExpirationMap;
 import org.cobbzilla.util.reflect.ReflectionUtil;
 import org.cobbzilla.wizard.model.Identifiable;
 import org.cobbzilla.wizard.model.RelatedEntities;
+import org.cobbzilla.wizard.model.SqlDefaultSearchField;
 import org.cobbzilla.wizard.model.entityconfig.annotations.ECSearchable;
 import org.cobbzilla.wizard.model.search.SearchField;
 import org.cobbzilla.wizard.model.search.SearchQuery;
@@ -23,6 +23,7 @@ import java.util.concurrent.ConcurrentHashMap;
 
 import static org.cobbzilla.util.daemon.ZillaRuntime.die;
 import static org.cobbzilla.util.daemon.ZillaRuntime.hashOf;
+import static org.cobbzilla.util.reflect.ReflectionUtil.fieldsWithAnnotation;
 import static org.cobbzilla.util.string.StringUtil.camelCaseToSnakeCase;
 import static org.cobbzilla.util.string.StringUtil.sqlFilter;
 
@@ -62,9 +63,9 @@ public interface SqlViewSearchableDAO<T extends Identifiable> extends DAO<T> {
     Map<String, String> _fieldCache = new ExpirationMap<>();
 
     default String buildBound(String bound, String value, List<Object> params, String locale) {
-        for (Field f : FieldUtils.getAllFields(getEntityClass())) {
+        for (Field f : fieldsWithAnnotation(getEntityClass(), ECSearchable.class)) {
+            if (!f.getName().equalsIgnoreCase(bound)) continue;
             final ECSearchable search = f.getAnnotation(ECSearchable.class);
-            if (!f.getName().equalsIgnoreCase(bound) || search == null) continue;
 
             final String hash = hashOf(f, search, bound, value, params, locale);
             return _fieldCache.computeIfAbsent(hash, k -> {

wizard-server/src/main/resources/org/cobbzilla/wizard/dao/default_search_view.sql.hbs

@@ -1,4 +1,4 @@
 CREATE OR REPLACE VIEW {{safeSql viewName}} ( {{#each viewColumns}}{{#unless @first}}, {{/unless}}{{safeSql this}}{{/each}} )
 AS SELECT {{#each selectColumns}}{{#unless @first}}, {{/unless}}{{safeSql this}}{{/each}}
 FROM {{#each fromClauses}}{{#unless @first}}, {{/unless}}{{safeSql this}}{{/each}}
-WHERE {{#each whereClauses}}{{#unless @first}} AND {{/unless}}( {{safeSql this}} ){{/each}};
+WHERE 1=1 {{#each whereClauses}} AND ( {{safeSql this}} ){{/each}};