serialize kwftypes via their real type

the kwftype may not be lazy-created yet
so store instead a reference to the primary type
and potentially rebuild the kwftype on-demand

fix #16471

Author: vtjnash

src/dump.c

@@ -529,6 +529,20 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt)
         tag = 4;
     else if (dt == jl_uint8_type)
         tag = 8;
+
+    if (strncmp(jl_symbol_name(dt->name->name), "#kw#", 4) == 0) {
+        /* XXX: yuck, but the auto-generated kw types from the serializer isn't a real type, so we *must* be very careful */
+        assert(tag == 0 || tag == 5 || tag == 6);
+        if (tag == 6) {
+            jl_methtable_t *mt = dt->name->mt;
+            jl_datatype_t *primarydt = (jl_datatype_t*)jl_get_global(mt->module, mt->name);
+            assert(jl_is_datatype(primarydt));
+            assert(jl_typeof(primarydt->name->mt->kwsorter) == (jl_value_t*)dt);
+            dt = primarydt;
+            tag = 9;
+        }
+    }
+
     writetag(s, (jl_value_t*)SmallDataType_tag);
     write_uint8(s, 0); // virtual size
     jl_serialize_value(s, (jl_value_t*)jl_datatype_type);
@@ -542,6 +556,11 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt)
         jl_serialize_value(s, dt->parameters);
         return;
     }
+    if (tag == 9) {
+        jl_serialize_value(s, dt);
+        return;
+    }
+
     size_t nf = jl_datatype_nfields(dt);
     write_uint16(s, nf);
     write_int32(s, dt->size);
@@ -994,37 +1013,25 @@ struct jl_serialize_methcache_from_mod_env {
     ios_t *s;
     jl_sym_t *name;
     jl_module_t *mod;
-    int8_t iskw;
 };
 
 static int jl_serialize_methcache_from_mod(jl_typemap_entry_t *ml, void *closure)
 {
     struct jl_serialize_methcache_from_mod_env *env = (struct jl_serialize_methcache_from_mod_env*)closure;
     if (module_in_worklist(ml->func.method->module)) {
-        jl_serialize_value(env->s, env->mod);
-        jl_serialize_value(env->s, env->name);
-        write_int8(env->s, env->iskw);
-        jl_serialize_value(env->s, ml->simplesig);
         jl_serialize_value(env->s, ml->func.method);
+        jl_serialize_value(env->s, ml->simplesig);
     }
     return 1;
 }
 
-static void jl_serialize_methtable_from_mod(ios_t *s, jl_typename_t *tn, int8_t iskw)
+static void jl_serialize_methtable_from_mod(ios_t *s, jl_typename_t *tn)
 {
     struct jl_serialize_methcache_from_mod_env env;
     env.s = s;
     env.mod = tn->module;
     env.name = tn->name;
-    env.iskw = iskw;
     assert(tn->module);
-    if (iskw) {
-        if (!tn->mt->kwsorter)
-            return;
-        assert(tn->mt->module == jl_gf_mtable(tn->mt->kwsorter)->module);
-        tn = ((jl_datatype_t*)jl_typeof(tn->mt->kwsorter))->name;
-        assert(tn->mt->kwsorter == NULL);
-    }
     jl_typemap_visitor(tn->mt->defs, jl_serialize_methcache_from_mod, &env);
 }
 
@@ -1037,14 +1044,12 @@ static void jl_serialize_lambdas_from_mod(ios_t *s, jl_module_t *m)
         if (table[i] != HT_NOTFOUND) {
             jl_binding_t *b = (jl_binding_t*)table[i];
             if (b->owner == m && b->value && b->constp) {
-                if (jl_is_datatype(b->value) &&
-                        (strlen(jl_symbol_name(b->name)) <= 4 || strncmp(jl_symbol_name(b->name), "#kw#", 4))) { /* XXX: yuck, but the auto-generated kw types from the serializer isn't a real type, so we *must* reject it */
+                if (jl_is_datatype(b->value)) {
                     jl_typename_t *tn = ((jl_datatype_t*)b->value)->name;
                     if (tn->module == m && tn->name == b->name) {
                         jl_methtable_t *mt = tn->mt;
                         if (mt != NULL && (jl_value_t*)mt != jl_nothing && (mt != jl_type_type_mt || tn == jl_type_type->name)) {
-                            jl_serialize_methtable_from_mod(s, tn, 0);
-                            jl_serialize_methtable_from_mod(s, tn, 1);
+                            jl_serialize_methtable_from_mod(s, tn);
                         }
                     }
                 }
@@ -1176,12 +1181,18 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc)
         backref_list.items[pos] = dtv;
         return dtv;
     }
+    if (tag == 9) {
+        jl_datatype_t *primarydt = (jl_datatype_t*)jl_deserialize_value(s, NULL);
+        jl_value_t *dtv = jl_typeof(jl_get_kwsorter(primarydt->name));
+        backref_list.items[pos] = dtv;
+        return dtv;
+    }
     uint16_t nf = read_uint16(s);
     size_t size = read_int32(s);
     uint8_t flags = read_uint8(s);
     uint8_t depth = read_int32(s);
     uint8_t fielddesc_type = read_int8(s);
-    jl_datatype_t *dt;
+    jl_datatype_t *dt = NULL;
     if (tag == 2)
         dt = jl_int32_type;
     else if (tag == 3)
@@ -1190,8 +1201,10 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc)
         dt = jl_int64_type;
     else if (tag == 8)
         dt = jl_uint8_type;
-    else
+    else if (tag == 0 || tag == 5)
         dt = jl_new_uninitialized_datatype(nf, fielddesc_type);
+    else
+        assert(0);
     assert(tree_literal_values==NULL && mode != MODE_AST);
     backref_list.items[pos] = dt;
     dt->size = size;
@@ -1686,19 +1699,12 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t
 static void jl_deserialize_lambdas_from_mod(ios_t *s)
 {
     while (1) {
-        jl_module_t *mod = (jl_module_t*)jl_deserialize_value(s, NULL);
-        if (mod == NULL)
+        jl_method_t *meth = (jl_method_t*)jl_deserialize_value(s, NULL);
+        if (meth == NULL)
             return;
-        jl_sym_t *name = (jl_sym_t*)jl_deserialize_value(s, NULL);
-        jl_datatype_t *gf = (jl_datatype_t*)jl_get_global(mod, name);
-        assert(jl_is_datatype(gf));
-        int8_t iskw = read_int8(s);
-        if (iskw) {
-            gf = (jl_datatype_t*)jl_typeof(jl_get_kwsorter(gf->name));
-            assert(jl_is_datatype(gf));
-        }
         jl_tupletype_t *simpletype = (jl_tupletype_t*)jl_deserialize_value(s, NULL);
-        jl_method_t *meth = (jl_method_t*)jl_deserialize_value(s, NULL);
+        jl_datatype_t *gf = (jl_datatype_t*)jl_tparam0(meth->sig);
+        assert(jl_is_datatype(gf));
         jl_method_table_insert(gf->name->mt, meth, simpletype);
     }
 }

test/compile.jl

@@ -4,7 +4,7 @@ using Base.Test
 
 function redirected_stderr()
     rd, wr = redirect_stderr()
-    @async readall(rd) # make sure the kernel isn't being forced to buffer the output
+    @async readstring(rd) # make sure the kernel isn't being forced to buffer the output
     nothing
 end
 
@@ -20,16 +20,28 @@ try
     write(Foo_file,
           """
           __precompile__(true)
+
+          # test that docs get reconnected
           module $Foo_module
-          @doc "foo function" foo(x) = x + 1
-          include_dependency("foo.jl")
-          include_dependency("foo.jl")
-          module Bar
-          @doc "bar function" bar(x) = x + 2
-          include_dependency("bar.jl")
-          end
+              @doc "foo function" foo(x) = x + 1
+              include_dependency("foo.jl")
+              include_dependency("foo.jl")
+              module Bar
+                  @doc "bar function" bar(x) = x + 2
+                  include_dependency("bar.jl")
+              end
+
+              # test that types and methods get reconnected correctly
+              # issue 16529 (adding a method to a type with no instances)
+              (::Task)(::UInt8, ::UInt16, ::UInt32) = 2
+
+              # issue 16471 (capturing references to an kwfunc)
+              Base.Test.@test_throws ErrorException Core.kwfunc(Base.nothing)
+              Base.nothing(::UInt8, ::UInt16, ::UInt32; x = 52) = x
+              const nothingkw = Core.kwfunc(Base.nothing)
           end
           """)
+    @test_throws ErrorException Core.kwfunc(Base.nothing) # make sure `nothing` didn't have a kwfunc (which would invalidate the attempted test)
 
     # Issue #12623
     @test __precompile__(true) === nothing
@@ -38,7 +50,7 @@ try
     cachefile = joinpath(dir, "$Foo_module.ji")
 
     # use _require_from_serialized to ensure that the test fails if
-    # the module doesn't load from the image:
+    # the module doesn't reload from the image:
     try
         redirected_stderr()
         @test nothing !== Base._require_from_serialized(myid(), Foo_module, #=broadcast-load=#false)
@@ -58,6 +70,11 @@ try
         @test sort(deps[1]) == map(s -> (s, Base.module_uuid(eval(s))),
                                    [:Base,:Core,:Main])
         @test map(x -> x[1], sort(deps[2])) == [Foo_file,joinpath(dir,"bar.jl"),joinpath(dir,"foo.jl")]
+
+        @test current_task()(0x01, 0x4000, 0x30031234) == 2
+        @test nothing(0x01, 0x4000, 0x30031234) == 52
+        @test nothing(0x01, 0x4000, 0x30031234; x = 9142) == 9142
+        @test Foo.nothingkw === Core.kwfunc(Base.nothing)
     end
 
     Baz_file = joinpath(dir, "Baz.jl")