Bit more cleanup wrt #2457

Author: cowtowncoder

src/main/java/com/fasterxml/jackson/databind/JavaType.java

@@ -6,6 +6,7 @@
 import com.fasterxml.jackson.core.type.ResolvedType;
 import com.fasterxml.jackson.databind.type.TypeBindings;
 import com.fasterxml.jackson.databind.type.TypeFactory;
+import com.fasterxml.jackson.databind.util.ClassUtil;
 
 /**
  * Base class for type token classes used both to contain information
@@ -290,8 +291,8 @@ public boolean isConcrete() {
     @Override
     public final boolean isEnumType() {
         // 29-Sep-2019, tatu: `Class.isEnum()` not enough to detect custom subtypes,
-        //    use this instead
-//        return Enum.class.isAssignableFrom(_class);
+        //   but for some reason this fix will break couple of unit tests:
+//        return ClassUtil.isEnumType(_class);
         return _class.isEnum();
     }
 

src/main/java/com/fasterxml/jackson/databind/jsontype/impl/ClassNameIdResolver.java

@@ -89,7 +89,7 @@ protected JavaType _typeFromId(String id, DatabindContext ctxt) throws IOExcepti
     protected String _idFrom(Object value, Class<?> cls, TypeFactory typeFactory)
     {
         // Need to ensure that "enum subtypes" work too
-        if (Enum.class.isAssignableFrom(cls)) {
+        if (ClassUtil.isEnumType(cls)) {
             if (!cls.isEnum()) { // means that it's sub-class of base enum, so:
                 cls = cls.getSuperclass();
             }

src/main/java/com/fasterxml/jackson/databind/module/SimpleDeserializers.java

@@ -126,6 +126,10 @@ public JsonDeserializer<?> findEnumDeserializer(Class<?> type,
         }
         JsonDeserializer<?> deser = _classMappings.get(new ClassKey(type));
         if (deser == null) {
+            // 29-Sep-2019, tatu: Not 100% sure this is workable logic but leaving
+            //   as is (wrt [databind#2457]. Probably works ok since this covers direct
+            //   sub-classes of `Enum`; but even if custom sub-classes aren't, unlikely
+            //   mapping for those ever requested for deserialization
             if (_hasEnumDeserializer && type.isEnum()) {
                 deser = _classMappings.get(new ClassKey(Enum.class));
             }

src/main/java/com/fasterxml/jackson/databind/ser/BasicSerializerFactory.java

@@ -421,7 +421,7 @@ protected final JsonSerializer<?> findSerializerByPrimaryType(SerializerProvider
             }
             return NumberSerializer.instance;
         }
-        if (Enum.class.isAssignableFrom(raw)) {
+        if (ClassUtil.isEnumType(raw) && raw != Enum.class) {
             return buildEnumSerializer(prov.getConfig(), type, beanDesc);
         }
         return null;

src/main/java/com/fasterxml/jackson/databind/ser/std/JsonValueSerializer.java

@@ -249,7 +249,7 @@ public void acceptJsonFormatVisitor(JsonFormatVisitorWrapper visitor, JavaType t
          */
         final JavaType type = _accessor.getType();
         Class<?> declaring = _accessor.getDeclaringClass();
-        if ((declaring != null) && declaring.isEnum()) {
+        if ((declaring != null) && ClassUtil.isEnumType(declaring)) {
             if (_acceptJsonFormatVisitorForEnum(visitor, typeHint, declaring)) {
                 return;
             }

src/main/java/com/fasterxml/jackson/databind/ser/std/StdKeySerializers.java

@@ -100,7 +100,7 @@ public static JsonSerializer<Object> getFallbackKeySerializer(SerializationConfi
             }
             // 29-Sep-2019, tatu: [databind#2457] can not use 'rawKeyType.isEnum()`, won't work
             //    for subtypes.
-            if (Enum.class.isAssignableFrom(rawKeyType)) {
+            if (ClassUtil.isEnumType(rawKeyType)) {
                 return EnumKeySerializer.construct(rawKeyType,
                         EnumValues.constructFromName(config, (Class<Enum<?>>) rawKeyType));
             }

src/main/java/com/fasterxml/jackson/databind/util/ClassUtil.java

@@ -169,7 +169,7 @@ public static String canBeABeanType(Class<?> type)
         if (type.isArray()) {
             return "array";
         }
-        if (type.isEnum()) {
+        if (Enum.class.isAssignableFrom(type)) {
             return "enum";
         }
         if (type.isPrimitive()) {
@@ -954,6 +954,16 @@ public static void checkAndFixAccess(Member member, boolean force)
     /**********************************************************
      */
 
+    /**
+     * Helper method that encapsulates reliable check on whether
+     * given raw type "is an Enum", that is, is or extends {@link java.lang.Enum}.
+     *
+     * @since 2.10.1
+     */
+    public static boolean isEnumType(Class<?> rawType) {
+        return Enum.class.isAssignableFrom(rawType);
+    }
+
     /**
      * Helper method that can be used to dynamically figure out
      * enumeration type of given {@link EnumSet}, without having

src/test/java/com/fasterxml/jackson/databind/ser/TestEnumSerialization.java

@@ -147,10 +147,8 @@ public void testSimple() throws Exception
 
     public void testEnumSet() throws Exception
     {
-        StringWriter sw = new StringWriter();
-        EnumSet<TestEnum> value = EnumSet.of(TestEnum.B);
-        MAPPER.writeValue(sw, value);
-        assertEquals("[\"B\"]", sw.toString());
+        final EnumSet<TestEnum> value = EnumSet.of(TestEnum.B);
+        assertEquals("[\"B\"]", MAPPER.writeValueAsString(value));
     }
 
     /**
@@ -160,9 +158,7 @@ public void testEnumSet() throws Exception
      */
     public void testEnumUsingToString() throws Exception
     {
-        StringWriter sw = new StringWriter();
-        MAPPER.writeValue(sw, AnnotatedTestEnum.C2);
-        assertEquals("\"c2\"", sw.toString());
+        assertEquals("\"c2\"", MAPPER.writeValueAsString(AnnotatedTestEnum.C2));
     }
 
     public void testSubclassedEnums() throws Exception