remove ast from the Method, but make unspecialized also valid and owner of the uninferred ast

also rename li->unspecialized to li->unspecialized_ducttape, to make clear that this field is a temporary workaround until the interpreter is fully-functional

Author: vtjnash

base/boot.jl

@@ -60,17 +60,16 @@
 #end
 
 #type LambdaInfo
-#    inferred_ast::Expr
+#    ast::Union{Expr, Vector{UInt8}}
 #    rettype::Any
 #    sparam_vals::SimpleVector
 #    specTypes::Any
-#    unspecialized::LambdaInfo
+#    unspecialized_ducttape::LambdaInfo
 #    def::Method
 #    pure::Bool
 #end
 
 #type Method
-#    ast::Expr
 #    sig::Type
 #    tvars::Expr
 #    sparam_syms::SimpleVector
@@ -317,7 +316,7 @@ TypeVar(n::Symbol, lb::ANY, ub::ANY, b::Bool) =
     ccall(:jl_new_typevar_, Any, (Any, Any, Any, Any), n, lb::Type, ub::Type, b)::TypeVar
 
 TypeConstructor(p::ANY, t::ANY) = ccall(:jl_new_type_constructor, Any, (Any, Any), p::SimpleVector, t::Type)
-LambdaInfo(ast::Method, sparam_vals::SimpleVector, specTypes::DataType) = ccall(:jl_new_lambda_info, Any, (Any, Any, Any), ast, sparam_vals, specTypes)
+LambdaInfo(ast::Method, sparam_vals::SimpleVector, specTypes::DataType) = ccall(:jl_spec_lambda_info, Any, (Any, Any, Any), ast.unspecialized, sparam_vals, specTypes)
 
 Void() = nothing
 

base/inference.jl

@@ -29,7 +29,7 @@ type VarInfo
     mod::Module
 end
 
-function VarInfo(linfo::LambdaInfo, ast=linfo.inferred_ast)
+function VarInfo(linfo::LambdaInfo, ast=linfo.def.unspecialized.ast)
     if !isa(ast,Expr)
         ast = ccall(:jl_uncompress_ast, Any, (Any,Any), linfo.def, ast)
     end
@@ -65,15 +65,15 @@ type EmptyCallStack
 end
 
 type CallStack
-    ast
+    meth::Method
     types::Type
     recurred::Bool
     cycleid::Int
     result
     prev::Union{EmptyCallStack,CallStack}
     sv::VarInfo
 
-    CallStack(ast, types::ANY, prev) = new(ast, types, false, 0, Bottom, prev)
+    CallStack(meth, types::ANY, prev) = new(meth, types, false, 0, Bottom, prev)
 end
 
 inference_stack = EmptyCallStack()
@@ -741,7 +741,7 @@ function abstract_call_gf_by_type(f::ANY, argtype::ANY, e)
         limit = false
         # look at the stack to detect recursive calls with growing argument lists
         while sp !== EmptyCallStack()
-            if linfo.def.ast === sp.ast && length(argtypes) > length(sp.types.parameters)
+            if linfo.def === sp.meth && length(argtypes) > length(sp.types.parameters)
                 limit = true; break
             end
             sp = sp.prev
@@ -1440,7 +1440,7 @@ function typeinf(linfo::LambdaInfo, atypes::ANY, sparams::SimpleVector, def::Met
     end
     # TODO: typeinf currently gets stuck without this
     if def.name === :abstract_interpret || def.name === :tuple_elim_pass || def.name === :abstract_call_gf
-        return (def.ast, Any)
+        return (def.unspecialized.ast, Any)
     end
 
     (fulltree, result, rec) = typeinf_uncached(linfo, atypes, sparams, def, curtype, cop, true)
@@ -1485,7 +1485,6 @@ tupletype_tail(t::ANY, n) = Tuple{t.parameters[n:end]...}
 
 # compute an inferred (optionally optimized) AST without global effects (i.e. updating the cache)
 function typeinf_uncached(linfo::LambdaInfo, atypes::ANY, sparams::SimpleVector, def::Method, curtype, cop, optimize)
-    ast0 = def.ast
     #if dbg
     #    print("typeinf ", def.name, " ", object_id(ast0), "\n")
     #end
@@ -1501,7 +1500,7 @@ function typeinf_uncached(linfo::LambdaInfo, atypes::ANY, sparams::SimpleVector,
 
     f = inference_stack
     while !isa(f,EmptyCallStack)
-        if is(f.ast,ast0)
+        if is(f.meth, def)
             # impose limit if we recur and the argument types grow beyond MAX_TYPE_DEPTH
             td = type_depth(atypes)
             if td > type_depth(f.types)
@@ -1540,7 +1539,7 @@ function typeinf_uncached(linfo::LambdaInfo, atypes::ANY, sparams::SimpleVector,
     # check for recursion
     f = inference_stack
     while !isa(f,EmptyCallStack)
-        if (is(f.ast,ast0) || f.ast==ast0) && typeseq(f.types, atypes)
+        if (is(f.meth, def) || f.meth.unspecialized.ast == def.unspecialized.ast) && typeseq(f.types, atypes)
             # return best guess so far
             (f::CallStack).recurred = true
             (f::CallStack).cycleid = CYCLE_ID
@@ -1567,7 +1566,7 @@ function typeinf_uncached(linfo::LambdaInfo, atypes::ANY, sparams::SimpleVector,
     if cop
         ast = ccall(:jl_prepare_ast, Any, (Any,), def)::Expr
     else
-        ast = linfo.inferred_ast
+        ast = linfo.ast
     end
 
     sv = VarInfo(linfo, ast)
@@ -1605,7 +1604,7 @@ function typeinf_uncached(linfo::LambdaInfo, atypes::ANY, sparams::SimpleVector,
     end
 
     # our stack frame
-    frame = CallStack(ast0, atypes, inference_stack)
+    frame = CallStack(def, atypes, inference_stack)
     frame.sv = sv
     inference_stack = frame
     frame.result = curtype
@@ -1810,7 +1809,7 @@ function typeinf_uncached(linfo::LambdaInfo, atypes::ANY, sparams::SimpleVector,
                             # for an example see test/libgit2.jl on 0.5-pre master
                             # around e.g. commit c072d1ce73345e153e4fddf656cda544013b1219
                             inference_stack = (inference_stack::CallStack).prev
-                            return (ast0, Any, false)
+                            return (def.unspecialized.ast, Any, false)
                         end
                     end
                     handler_at[l] = cur_hand
@@ -2339,7 +2338,7 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atype::ANY, sv::VarInfo, enclosing
     #     # check call stack to see if this argument list is growing
     #     st = inference_stack
     #     while !isa(st, EmptyCallStack)
-    #         if st.ast === linfo.def.ast && length(atypes) > length(st.types)
+    #         if st.meth === linfo.def && length(atypes) > length(st.types)
     #             atypes = limit_tuple_type(atypes)
     #             meth = _methods(f, atypes, 1)
     #             if meth === false || length(meth) != 1

base/reflection.jl

@@ -245,9 +245,8 @@ done(mt::MethodTable, m::Method) = false
 done(mt::MethodTable, i::Void) = true
 
 uncompressed_ast(l::LambdaInfo) =
-    isa(l.inferred_ast,Expr) ? l.inferred_ast : ccall(:jl_uncompress_ast, Any, (Any,Any), l.def, l.inferred_ast)
-uncompressed_ast(l::Method) =
-    isa(l.ast,Expr) ? l.ast : ccall(:jl_uncompress_ast, Any, (Any,Any), l, l.ast)
+    isa(l.ast, Expr) ? l.ast : ccall(:jl_uncompress_ast, Any, (Any,Any), l.def, l.ast)
+uncompressed_ast(l::Method) = uncompressed_ast(l.unspecialized)
 
 # Printing code representations in IR and assembly
 function _dump_function(f, t::ANY, native, wrapper, strip_ir_metadata, dump_module)

base/serialize.jl

@@ -19,7 +19,7 @@ const TAGS = Any[
     #LongSymbol, LongTuple, LongExpr,
     Symbol, Tuple, Expr,  # dummy entries, intentionally shadowed by earlier ones
     LineNumberNode, SymbolNode, LabelNode, GotoNode,
-    QuoteNode, TopNode, TypeVar, Box, Method,
+    QuoteNode, TopNode, TypeVar, Box, :reserved00,
     Module, #=UndefRefTag=#Symbol, Task, ASCIIString, UTF8String,
     UTF16String, UTF32String, Float16,
     SimpleVector, #=BackrefTag=#Symbol, :reserved11, :reserved12,
@@ -72,7 +72,6 @@ const BACKREF_TAG = Int32(sertag(SimpleVector)+1)
 const EXPR_TAG = sertag(Expr)
 const LONGEXPR_TAG = Int32(sertag(Expr)+3)
 const MODULE_TAG = sertag(Module)
-const METHODDATA_TAG = sertag(Method)
 const TASK_TAG = sertag(Task)
 const DATATYPE_TAG = sertag(DataType)
 const TYPENAME_TAG = sertag(TypeName)

src/alloc.c

@@ -280,16 +280,14 @@ JL_DLLEXPORT jl_method_info_t *jl_new_method_info(jl_value_t *ast, jl_svec_t *tv
     jl_method_info_t *li =
         (jl_method_info_t*)newobj((jl_value_t*)jl_method_info_type,
                                   NWORDS(sizeof(jl_method_info_t)));
-    li->ast = ast;
+    JL_GC_PUSH1(&li);
     li->sig = NULL;
     li->tvars = NULL;
     li->next = NULL;
     li->file = null_sym;
     li->line = 0;
-    li->pure = 0;
     li->isstaged = 0;
     li->va = 0;
-    li->called = 0xff;
     li->module = ctx;
     li->sparam_syms = tvars;
     li->tfunc = jl_nothing;
@@ -299,10 +297,11 @@ JL_DLLEXPORT jl_method_info_t *jl_new_method_info(jl_value_t *ast, jl_svec_t *tv
     li->invokes = (jl_methtable_t*)jl_nothing;
     li->next = (jl_method_info_t*)jl_nothing;
     li->unspecialized = NULL;
+    uint8_t pure = 0, called = 0xff;
     if (ast != NULL && jl_is_expr(ast)) {
         jl_array_t *body = jl_lam_body((jl_expr_t*)ast)->args;
         if (has_meta(body, pure_sym))
-            li->pure = 1;
+            pure = 1;
         jl_value_t *body1 = skip_meta(body);
         if (jl_is_linenode(body1)) {
             li->file = jl_linenode_file(body1);
@@ -312,10 +311,10 @@ JL_DLLEXPORT jl_method_info_t *jl_new_method_info(jl_value_t *ast, jl_svec_t *tv
             li->file = (jl_sym_t*)jl_exprarg(body1, 1);
             li->line = jl_unbox_long(jl_exprarg(body1, 0));
         }
-        jl_array_t *vis = jl_lam_vinfo((jl_expr_t*)li->ast);
-        jl_array_t *args = jl_lam_args((jl_expr_t*)li->ast);
+        jl_array_t *vis = jl_lam_vinfo((jl_expr_t*)ast);
+        jl_array_t *args = jl_lam_args((jl_expr_t*)ast);
         size_t narg = jl_array_len(args);
-        uint8_t called=0;
+        called=0;
         int i, j=0;
         for(i=1; i < narg && i <= 8; i++) {
             jl_value_t *ai = jl_cellref(args,i);
@@ -328,38 +327,68 @@ JL_DLLEXPORT jl_method_info_t *jl_new_method_info(jl_value_t *ast, jl_svec_t *tv
             if (jl_unbox_long(jl_cellref(vj,2))&64)
                 called |= (1<<(i-1));
         }
-        li->called = called;
     }
+
+    jl_lambda_info_t *unspec = jl_spec_lambda_info(NULL, jl_emptysvec, (jl_tupletype_t*)jl_anytuple_type);
+    unspec->ast = ast;
+    unspec->pure = pure;
+    unspec->called = called;
+    unspec->def = li;
+    li->unspecialized = unspec;
+    jl_gc_wb(li, unspec);
+    JL_GC_POP();
     return li;
 }
 
-
-JL_DLLEXPORT jl_lambda_info_t *jl_new_lambda_info(jl_method_info_t *ast, jl_svec_t *sparam_vals, jl_datatype_t *specTypes)
+// return a new lambda-info that has some extra static parameters merged in.
+JL_DLLEXPORT jl_lambda_info_t *jl_spec_lambda_info(jl_lambda_info_t *li, jl_svec_t *sp, jl_tupletype_t *types)
 {
-    jl_lambda_info_t *li =
+    assert((li && jl_svec_len(li->def->sparam_syms) == jl_svec_len(sp)) || sp == jl_emptysvec);
+    jl_lambda_info_t *nli =
         (jl_lambda_info_t*)newobj((jl_value_t*)jl_lambda_info_type,
                                   NWORDS(sizeof(jl_lambda_info_t)));
-    li->inferred_ast = ast->ast;
-    li->rettype = (jl_value_t*)jl_any_type;
-    li->pure = ast->pure;
-    li->called = ast->called;
-    li->sparam_vals = sparam_vals;
-    li->fptr = NULL;
-    li->jlcall_api = 0;
-    li->functionObjects.functionObject = NULL;
-    li->functionObjects.specFunctionObject = NULL;
-    li->functionObjects.cFunctionList = NULL;
-    li->functionID = 0;
-    li->specFunctionID = 0;
-    li->specTypes = specTypes;
-    li->inferred = 0;
-    li->inInference = 0;
-    li->inCompile = 0;
-    li->unspecialized = NULL;
-    li->def = ast;
-    return li;
+    nli->sparam_vals = sp;
+    nli->specTypes = types;
+
+    // clone the basic metadata
+    nli->ast = li ? li->ast : NULL;
+    nli->def = li ? li->def : NULL;
+    nli->rettype = li ? li->rettype : (jl_value_t*)jl_any_type;
+    nli->pure = li ? li->pure : 0;
+    nli->called = li ? li->called : 0xff;
+
+    // some values never get cloned
+    nli->inInference = 0;
+    nli->inCompile = 0;
+    nli->unspecialized_ducttape = NULL;
+    nli->inferred = 0;
+    if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_OFF && li != NULL) {
+        // copy fptr from the unspecialized method definition
+        nli->fptr = li->fptr;
+        nli->jlcall_api = li->jlcall_api;
+        nli->functionObjects.functionObject = li->functionObjects.functionObject;
+        nli->functionObjects.specFunctionObject = li->functionObjects.specFunctionObject;
+        nli->functionID = li->functionID;
+        nli->specFunctionID = li->functionID;
+        if (nli->fptr == NULL) {
+            jl_printf(JL_STDERR,"code missing for ");
+            jl_static_show(JL_STDERR, (jl_value_t*)nli);
+            jl_printf(JL_STDERR, "  sysimg may not have been built with --compile=all\n");
+        }
+    }
+    else {
+        nli->fptr = NULL;
+        nli->jlcall_api = 0;
+        nli->functionObjects.functionObject = NULL;
+        nli->functionObjects.specFunctionObject = NULL;
+        nli->functionObjects.cFunctionList = NULL;
+        nli->functionID = 0;
+        nli->specFunctionID = 0;
+    }
+    return nli;
 }
 
+
 // symbols --------------------------------------------------------------------
 
 JL_DEFINE_MUTEX(symbol_table)

src/ast.c

@@ -484,7 +484,7 @@ static jl_value_t *scm_to_julia_(fl_context_t *fl_ctx, value_t e, int eo)
                 e = cdr_(e);
             }
             nli = jl_new_method_info((jl_value_t*)ex, tvars, jl_current_module);
-            jl_preresolve_globals(nli->ast, nli);
+            jl_preresolve_globals(nli->unspecialized->ast, nli);
             JL_GC_POP();
             return (jl_value_t*)nli;
         }
@@ -914,11 +914,9 @@ jl_array_t *jl_lam_args(jl_expr_t *l)
 
 jl_sym_t *jl_lam_argname(jl_method_info_t *li, int i)
 {
-    jl_expr_t *ast;
-    if (jl_is_expr(li->ast))
-        ast = (jl_expr_t*)li->ast;
-    else
-        ast = (jl_expr_t*)jl_uncompress_ast(li, li->ast);
+    jl_expr_t *ast = (jl_expr_t*)li->unspecialized->ast;
+    if (!jl_is_expr(ast))
+        ast = (jl_expr_t*)jl_uncompress_ast(li, (jl_value_t*)ast);
     // NOTE (gc root): `ast` is not rooted here, but jl_lam_args and jl_cellref
     // do not allocate.
     return (jl_sym_t*)jl_cellref(jl_lam_args(ast),i);
@@ -1091,8 +1089,7 @@ static jl_value_t *dont_copy_ast(jl_value_t *expr)
 // this tree can then be further mutated by optimization passes.
 JL_DLLEXPORT jl_value_t *jl_prepare_ast(jl_method_info_t *li)
 {
-    jl_value_t *ast = li->ast;
-    if (ast == NULL) return NULL;
+    jl_value_t *ast = li->unspecialized->ast;
     JL_GC_PUSH1(&ast);
     if (!jl_is_expr(ast)) {
         ast = jl_uncompress_ast(li, ast);
@@ -1169,7 +1166,7 @@ static int jl_in_sym_svec(jl_svec_t *a, jl_sym_t *v)
 
 int jl_local_in_linfo(jl_method_info_t *linfo, jl_sym_t *sym)
 {
-    return jl_in_vinfo_array(jl_lam_vinfo((jl_expr_t*)linfo->ast), sym) ||
+    return jl_in_vinfo_array(jl_lam_vinfo((jl_expr_t*)linfo->unspecialized->ast), sym) ||
         jl_in_sym_svec(linfo->sparam_syms, sym);
 }
 
@@ -1185,7 +1182,7 @@ jl_value_t *jl_preresolve_globals(jl_value_t *expr, jl_method_info_t *lam)
     }
     else if (jl_is_method_info(expr)) {
         jl_method_info_t *l = (jl_method_info_t*)expr;
-        (void)jl_preresolve_globals(l->ast, l);
+        (void)jl_preresolve_globals(l->unspecialized->ast, l);
     }
     else if (jl_is_expr(expr)) {
         jl_expr_t *e = (jl_expr_t*)expr;

src/builtins.c

@@ -1086,19 +1086,18 @@ jl_value_t *jl_mk_builtin_func(const char *name, jl_fptr_t fptr)
     jl_value_t *f = jl_new_generic_function_with_supertype(sname, jl_core_module, jl_builtin_type, 0);
     jl_method_info_t *li = jl_new_method_info(jl_nothing, jl_emptysvec, jl_core_module);
     li->name = sname;
+    li->unspecialized->fptr = fptr;
     // TODO jb/functions: what should li->ast be?
-    li->ast = (jl_value_t*)jl_exprn(lambda_sym,0); jl_gc_wb(li, li->ast);
-    jl_lambda_info_t *unspec = jl_new_lambda_info(li, jl_emptysvec, jl_anytuple_type);
-    li->unspecialized = unspec;
-    unspec->fptr = fptr;
-    jl_method_cache_insert(jl_gf_mtable(f), jl_anytuple_type, unspec);
+    li->unspecialized->ast = (jl_value_t*)jl_exprn(lambda_sym,0);
+    jl_gc_wb(li->unspecialized, li->unspecialized->ast);
+    jl_method_cache_insert(jl_gf_mtable(f), jl_anytuple_type, li->unspecialized);
     return f;
 }
 
 jl_fptr_t jl_get_builtin_fptr(jl_value_t *b)
 {
     assert(jl_subtype(b, (jl_value_t*)jl_builtin_type, 1));
-    return ((jl_method_info_t*)jl_gf_mtable(b)->cache->func)->unspecialized->fptr;
+    return jl_gf_mtable(b)->cache->func->fptr;
 }
 
 static void add_builtin_func(const char *name, jl_fptr_t fptr)

src/codegen.cpp

@@ -4205,7 +4205,7 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio
     assert(definitions && "Capturing definitions is always required");
 
     // step 1. unpack AST and allocate codegen context for this function
-    jl_expr_t *ast = (jl_expr_t*)lam->inferred_ast;
+    jl_expr_t *ast = (jl_expr_t*)lam->ast;
     JL_GC_PUSH1(&ast);
     if (!jl_is_expr(ast)) {
         ast = (jl_expr_t*)jl_uncompress_ast(lam->def, (jl_value_t*)ast);