More work for #888, simple custom filter appears to work now.

src/main/java/com/fasterxml/jackson/databind/introspect/JacksonAnnotationIntrospector.java

@@ -723,31 +723,31 @@ public JsonInclude.Include findSerializationInclusionForContent(Annotated a, Jso
     public JsonInclude.Value findPropertyInclusion(Annotated a)
     {
         JsonInclude inc = _findAnnotation(a, JsonInclude.class);
-        JsonInclude.Include valueIncl = (inc == null) ? JsonInclude.Include.USE_DEFAULTS : inc.value();
-        if (valueIncl == JsonInclude.Include.USE_DEFAULTS) {
+        JsonInclude.Value value = (inc == null) ? JsonInclude.Value.empty() : JsonInclude.Value.from(inc);
+
+        // only consider deprecated variant if we didn't have non-deprecated one:
+        if (value.getValueInclusion() == JsonInclude.Include.USE_DEFAULTS) {
             JsonSerialize ann = _findAnnotation(a, JsonSerialize.class);
             if (ann != null) {
-                JsonSerialize.Inclusion i2 = ann.include();
-                switch (i2) {
+                switch (ann.include()) {
                 case ALWAYS:
-                    valueIncl = JsonInclude.Include.ALWAYS;
+                    value = value.withValueInclusion(JsonInclude.Include.ALWAYS);
                     break;
                 case NON_NULL:
-                    valueIncl = JsonInclude.Include.NON_NULL;
+                    value = value.withValueInclusion(JsonInclude.Include.NON_NULL);
                     break;
                 case NON_DEFAULT:
-                    valueIncl = JsonInclude.Include.NON_DEFAULT;
+                    value = value.withValueInclusion(JsonInclude.Include.NON_DEFAULT);
                     break;
                 case NON_EMPTY:
-                    valueIncl = JsonInclude.Include.NON_EMPTY;
+                    value = value.withValueInclusion(JsonInclude.Include.NON_EMPTY);
                     break;
                 case DEFAULT_INCLUSION:
                 default:
                 }
             }
         }
-        JsonInclude.Include contentIncl = (inc == null) ? JsonInclude.Include.USE_DEFAULTS : inc.content();
-        return JsonInclude.Value.construct(valueIncl, contentIncl);
+        return value;
     }
 
     @Override

src/main/java/com/fasterxml/jackson/databind/ser/DefaultSerializerProvider.java

@@ -149,7 +149,7 @@ public Object includeFilterInstance(BeanPropertyDefinition forProperty,
         HandlerInstantiator hi = _config.getHandlerInstantiator();
         Object filter = (hi == null) ? null : hi.includeFilterInstance(_config, forProperty, filterClass);
         if (filter == null) {
-            filter = (JsonSerializer<?>) ClassUtil.createInstance(filterClass,
+            filter = ClassUtil.createInstance(filterClass,
                     _config.canOverrideAccessModifiers());
         }
         return filter;

src/main/java/com/fasterxml/jackson/databind/ser/PropertyBuilder.java

@@ -140,11 +140,10 @@ protected BeanPropertyWriter buildWriter(SerializerProvider prov,
 
         inclV = inclV.withOverrides(propDef.findInclusion());
         JsonInclude.Include inclusion = inclV.getValueInclusion();
-
         if (inclusion == JsonInclude.Include.USE_DEFAULTS) { // should not occur but...
             inclusion = JsonInclude.Include.ALWAYS;
         }
-        
+
         switch (inclusion) {
         case NON_DEFAULT:
             // 11-Nov-2015, tatu: This is tricky because semantics differ between cases,
@@ -186,6 +185,7 @@ protected BeanPropertyWriter buildWriter(SerializerProvider prov,
             break;
         case CUSTOM: // new with 2.9
             valueToSuppress = prov.includeFilterInstance(propDef, inclV.getValueFilter());
+
             if (valueToSuppress == null) { // is this legal?
                 suppressNulls = true;
             } else {
@@ -194,9 +194,10 @@ protected BeanPropertyWriter buildWriter(SerializerProvider prov,
                 try {
                     suppressNulls = valueToSuppress.equals(null);
                 } catch (Throwable t) {
-                    prov.reportBadDefinition(_beanDesc.getType(),
-                            "Problem determining whether `null` values are to be suppressed: "+t.getMessage(),
-                            t);
+                    String msg = String.format(
+"Problem determining whether filter of type '%s' should filter out `null` values: (%s) %s",
+valueToSuppress.getClass().getName(), t.getClass().getName(), t.getMessage());
+                    prov.reportBadDefinition(_beanDesc.getType(), msg, t);
                 }
             }
             break;

src/test/java/com/fasterxml/jackson/databind/filter/JsonIncludeCustomTest.java

@@ -1,21 +1,78 @@
 package com.fasterxml.jackson.databind.filter;
 
+import com.fasterxml.jackson.annotation.*;
+
 import com.fasterxml.jackson.databind.BaseMapTest;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.exc.InvalidDefinitionException;
 
 // Tests for [databind#888]
 public class JsonIncludeCustomTest extends BaseMapTest
 {
+    static class FooFilter {
+        @Override
+        public boolean equals(Object other) {
+            if (other == null) { // do NOT filter out nulls
+                return false;
+            }
+            // in fact, only filter out exact String "foo"
+            return "foo".equals(other);
+        }
+    }
+
+    // for testing prob with `equals(null)` which SHOULD be allowed
+    static class BrokenFilter {
+        @Override
+        public boolean equals(Object other) {
+            /*String str = */ other.toString();
+            return false;
+        }
+    }
+
+    static class FooBean {
+        @JsonInclude(value=JsonInclude.Include.CUSTOM,
+                valueFilter=FooFilter.class)
+        public String value;
+
+        public FooBean(String v) { value = v; }
+    }
+
+    static class BrokenBean {
+        @JsonInclude(value=JsonInclude.Include.CUSTOM,
+                valueFilter=BrokenFilter.class)
+        public String value;
+
+        public BrokenBean(String v) { value = v; }
+    }
+
     /*
     /**********************************************************
-    /* Test methods
+    /* Test methods, success
     /**********************************************************
      */
 
     final private ObjectMapper MAPPER = new ObjectMapper();
 
     public void testSimpleCustomFilter() throws Exception
     {
-
+        assertEquals(aposToQuotes("{'value':'x'}"), MAPPER.writeValueAsString(new FooBean("x")));
+        assertEquals("{}", MAPPER.writeValueAsString(new FooBean("foo")));
+    }
+
+    /*
+    /**********************************************************
+    /* Test methods, fail handling
+    /**********************************************************
+     */
+
+    public void testBrokenFilter() throws Exception
+    {
+        try {
+            String json = MAPPER.writeValueAsString(new BrokenBean("foo"));
+            fail("Should not pass, produced: "+json);
+        } catch (InvalidDefinitionException e) {
+            verifyException(e, "Problem determining whether filter of type");
+            verifyException(e, "filter out `null`");
+        }
     }
 }