Merge branch 'master' into kc/pidlock_repl_history

Author: mbauman

.gitignore

@@ -11,6 +11,8 @@
 /julia-*
 /source-dist.tmp
 /source-dist.tmp1
+/test/results_*.json
+/test/results_*.dat
 
 *.expmap
 *.exe

CONTRIBUTING.md

@@ -4,37 +4,40 @@ Hi! If you are new to the Julia community: welcome, and thanks for trying Julia.
 
 If you are already familiar with Julia itself, this blog post by Katharine Hyatt on [Making your first Julia pull request](https://kshyatt.github.io/post/firstjuliapr/) is a great way to get started.
 
-## Learning Julia
-
-[The learning page](https://julialang.org/learning) has a great list of resources for new and experienced users alike.
-
-## Before filing an issue
-
-- Reporting a potential bug? Please read the "[How to file a bug report](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#how-to-file-a-bug-report)" section to make sure that all necessary information is included.
 
-- Contributing code? Be sure to review the [contributor checklist](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#contributor-checklist) for helpful tips on the tools we use to build Julia.
+# Table of Contents
 
-- Library feature requests are generally not accepted on this issue tracker. New libraries should be developed as [packages](https://julialang.github.io/Pkg.jl/v1/creating-packages/). Discuss ideas for libraries at the [Julia Discourse forum](https://discourse.julialang.org). Doing so will often lead to pointers to existing projects and bring together collaborators with common interests.
+1. [Learning Julia](#learning-julia)
+2. [Filing an issue](#filing-an-issue)
+    - [Before filing an issue](#before-filing-an-issue)
+    - [How to file a bug report](#how-to-file-a-bug-report)
+3. [Submitting contributions](#submitting-contributions)
+    - [Contributor Checklist](#contributor-checklist)
+    - [Writing tests](#writing-tests)
+    - [Improving documentation](#improving-documentation)
+    - [Contributing to core functionality or base libraries](#contributing-to-core-functionality-or-base-libraries)
+    - [Contributing to the standard library](#contributing-to-the-standard-library)
+    - [Contributing to patch releases](#contributing-to-patch-releases)
+    - [Code Formatting Guidelines](#code-formatting-guidelines)
+    - [Git Recommendations For Pull Requests](#git-recommendations-for-pull-requests)
+4. [Resources](#resources)
 
-## Contributor Checklist
-
-* Create a [GitHub account](https://github.com/signup/free).
 
-* [Fork Julia](https://github.com/JuliaLang/julia/fork).
+## Learning Julia
 
-* Build the software and libraries (the first time takes a while, but it's fast after that). Detailed build instructions are in the [README](https://github.com/JuliaLang/julia/tree/master/README.md). Julia depends on several external packages; most are automatically downloaded and installed, but are less frequently updated than Julia itself.
+[The learning page](https://julialang.org/learning) has a great list of resources for new and experienced users alike.
 
-* Keep Julia current. Julia is a fast-moving target, and many details of the language are still settling out. Keep the repository up-to-date and rebase work-in-progress frequently to make merges simpler.
+## Filing an issue
 
-* Learn to use [git](https://git-scm.com), the version control system used by GitHub and the Julia project. Try a tutorial such as the one [provided by GitHub](https://try.GitHub.io/levels/1/challenges/1).
+### Before filing an issue
 
-* Review discussions on the [Julia Discourse forum](https://discourse.julialang.org).
+- Reporting a potential bug? Please read the "[How to file a bug report](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#how-to-file-a-bug-report)" section to make sure that all necessary information is included.
 
-* For more detailed tips, read the [submission guide](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#submitting-contributions) below.
+- Contributing code? Be sure to review the [contributor checklist](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#contributor-checklist) for helpful tips on the tools we use to build Julia.
 
-* Relax and have fun!
+- Library feature requests are generally not accepted on this issue tracker. New libraries should be developed as [packages](https://julialang.github.io/Pkg.jl/v1/creating-packages/). Discuss ideas for libraries at the [Julia Discourse forum](https://discourse.julialang.org). Doing so will often lead to pointers to existing projects and bring together collaborators with common interests.
 
-## How to file a bug report
+### How to file a bug report
 
 A useful bug report filed as a GitHub issue provides information about how to reproduce the error.
 
@@ -56,6 +59,24 @@ A useful bug report filed as a GitHub issue provides information about how to re
 
 ## Submitting contributions
 
+### Contributor Checklist
+
+* Create a [GitHub account](https://github.com/signup/free).
+
+* [Fork Julia](https://github.com/JuliaLang/julia/fork).
+
+* Build the software and libraries (the first time takes a while, but it's fast after that). Detailed build instructions are in the [README](https://github.com/JuliaLang/julia/tree/master/README.md). Julia depends on several external packages; most are automatically downloaded and installed, but are less frequently updated than Julia itself.
+
+* Keep Julia current. Julia is a fast-moving target, and many details of the language are still settling out. Keep the repository up-to-date and rebase work-in-progress frequently to make merges simpler.
+
+* Learn to use [git](https://git-scm.com), the version control system used by GitHub and the Julia project. Try a tutorial such as the one [provided by GitHub](https://try.GitHub.io/levels/1/challenges/1).
+
+* Review discussions on the [Julia Discourse forum](https://discourse.julialang.org).
+
+* For more detailed tips, read the [submission guide](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#submitting-contributions) below.
+
+* Relax and have fun!
+
 ### Writing tests
 
 There are never enough tests. Track [code coverage at Codecov](https://codecov.io/github/JuliaLang/julia), and help improve it.

Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl

@@ -1,46 +1,74 @@
 module CompilerDevTools
 
 using Compiler
+using Compiler: argextype, widenconst
 using Core.IR
+using Base: isexpr
+
+mutable struct SplitCacheOwner end
 
-struct SplitCacheOwner; end
 struct SplitCacheInterp <: Compiler.AbstractInterpreter
     world::UInt
+    owner::SplitCacheOwner
     inf_params::Compiler.InferenceParams
     opt_params::Compiler.OptimizationParams
     inf_cache::Vector{Compiler.InferenceResult}
     codegen_cache::IdDict{CodeInstance,CodeInfo}
     function SplitCacheInterp(;
         world::UInt = Base.get_world_counter(),
+        owner::SplitCacheOwner = SplitCacheOwner(),
         inf_params::Compiler.InferenceParams = Compiler.InferenceParams(),
         opt_params::Compiler.OptimizationParams = Compiler.OptimizationParams(),
         inf_cache::Vector{Compiler.InferenceResult} = Compiler.InferenceResult[])
-        new(world, inf_params, opt_params, inf_cache, IdDict{CodeInstance,CodeInfo}())
+        new(world, owner, inf_params, opt_params, inf_cache, IdDict{CodeInstance,CodeInfo}())
     end
 end
 
 Compiler.InferenceParams(interp::SplitCacheInterp) = interp.inf_params
 Compiler.OptimizationParams(interp::SplitCacheInterp) = interp.opt_params
 Compiler.get_inference_world(interp::SplitCacheInterp) = interp.world
 Compiler.get_inference_cache(interp::SplitCacheInterp) = interp.inf_cache
-Compiler.cache_owner(::SplitCacheInterp) = SplitCacheOwner()
+Compiler.cache_owner(interp::SplitCacheInterp) = interp.owner
 Compiler.codegen_cache(interp::SplitCacheInterp) = interp.codegen_cache
 
 import Core.OptimizedGenerics.CompilerPlugins: typeinf, typeinf_edge
-@eval @noinline typeinf(::SplitCacheOwner, mi::MethodInstance, source_mode::UInt8) =
-    Base.invoke_in_world(which(typeinf, Tuple{SplitCacheOwner, MethodInstance, UInt8}).primary_world, Compiler.typeinf_ext_toplevel, SplitCacheInterp(; world=Base.tls_world_age()), mi, source_mode)
+@eval @noinline typeinf(owner::SplitCacheOwner, mi::MethodInstance, source_mode::UInt8) =
+    Base.invoke_in_world(which(typeinf, Tuple{SplitCacheOwner, MethodInstance, UInt8}).primary_world, Compiler.typeinf_ext_toplevel, SplitCacheInterp(; world=Base.tls_world_age(), owner), mi, source_mode)
 
-@eval @noinline function typeinf_edge(::SplitCacheOwner, mi::MethodInstance, parent_frame::Compiler.InferenceState, world::UInt, source_mode::UInt8)
+@eval @noinline function typeinf_edge(owner::SplitCacheOwner, mi::MethodInstance, parent_frame::Compiler.InferenceState, world::UInt, source_mode::UInt8)
     # TODO: This isn't quite right, we're just sketching things for now
-    interp = SplitCacheInterp(; world)
+    interp = SplitCacheInterp(; world, owner)
     Compiler.typeinf_edge(interp, mi.def, mi.specTypes, Core.svec(), parent_frame, false, false)
 end
 
-function with_new_compiler(f, args...)
-    mi = @ccall jl_method_lookup(Any[f, args...]::Ptr{Any}, (1+length(args))::Csize_t, Base.tls_world_age()::Csize_t)::Ref{Core.MethodInstance}
-    world = Base.tls_world_age()
+function lookup_method_instance(f, args...)
+    @ccall jl_method_lookup(Any[f, args...]::Ptr{Any}, (1+length(args))::Csize_t, Base.tls_world_age()::Csize_t)::Ref{Core.MethodInstance}
+end
+
+function Compiler.optimize(interp::SplitCacheInterp, opt::Compiler.OptimizationState, caller::Compiler.InferenceResult)
+    @invoke Compiler.optimize(interp::Compiler.AbstractInterpreter, opt::Compiler.OptimizationState, caller::Compiler.InferenceResult)
+    ir = opt.result.ir::Compiler.IRCode
+    override = GlobalRef(@__MODULE__(), :with_new_compiler)
+    for inst in ir.stmts
+        stmt = inst[:stmt]
+        isexpr(stmt, :call) || continue
+        f = stmt.args[1]
+        f === override && continue
+        if isa(f, GlobalRef)
+            T = widenconst(argextype(f, ir))
+            T <: Core.Builtin && continue
+        end
+        insert!(stmt.args, 1, override)
+        insert!(stmt.args, 3, interp.owner)
+    end
+end
+
+with_new_compiler(f, args...; owner::SplitCacheOwner = SplitCacheOwner()) = with_new_compiler(f, owner, args...)
+
+function with_new_compiler(f, owner::SplitCacheOwner, args...)
+    mi = lookup_method_instance(f, args...)
     new_compiler_ci = Core.OptimizedGenerics.CompilerPlugins.typeinf(
-        SplitCacheOwner(), mi, Compiler.SOURCE_MODE_ABI
+        owner, mi, Compiler.SOURCE_MODE_ABI
     )
     invoke(f, new_compiler_ci, args...)
 end

Compiler/extras/CompilerDevTools/test/runtests.jl

@@ -0,0 +1,20 @@
+using Test
+using Compiler: code_cache
+using Base: inferencebarrier
+using CompilerDevTools
+using CompilerDevTools: lookup_method_instance, SplitCacheInterp
+
+@testset "CompilerDevTools" begin
+  do_work(x, y) = x + y
+  f1() = do_work(inferencebarrier(1), inferencebarrier(2))
+  interp = SplitCacheInterp()
+  cache = code_cache(interp)
+  mi = lookup_method_instance(f1)
+  @test !haskey(cache, mi)
+  @test with_new_compiler(f1, interp.owner) === 3
+  @test haskey(cache, mi)
+  # Here `do_work` is compiled at runtime, and so must have
+  # required extra work to be cached under the same cache owner.
+  mi = lookup_method_instance(do_work, 1, 2)
+  @test haskey(cache, mi)
+end;

Compiler/extras/CompilerDevTools/test/testpkg.jl

@@ -0,0 +1,6 @@
+using Pkg
+
+Pkg.activate(dirname(@__DIR__)) do
+  Pkg.instantiate()
+  include("runtests.jl")
+end

Compiler/src/Compiler.jl

@@ -49,7 +49,8 @@ using Core: ABIOverride, Builtin, CodeInstance, IntrinsicFunction, MethodInstanc
 
 using Base
 using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospecializeinfer,
-    BINDING_KIND_GLOBAL, BINDING_KIND_UNDEF_CONST, BINDING_KIND_BACKDATED_CONST, BINDING_KIND_DECLARED,
+    PARTITION_KIND_GLOBAL, PARTITION_KIND_UNDEF_CONST, PARTITION_KIND_BACKDATED_CONST, PARTITION_KIND_DECLARED,
+    PARTITION_FLAG_DEPWARN,
     Base, BitVector, Bottom, Callable, DataTypeFieldDesc,
     EffectsOverride, Filter, Generator, IteratorSize, JLOptions, NUM_EFFECTS_OVERRIDES,
     OneTo, Ordering, RefValue, SizeUnknown, _NAMEDTUPLE_NAME,
@@ -68,7 +69,7 @@ using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospeciali
     structdiff, tls_world_age, unconstrain_vararg_length, unionlen, uniontype_layout,
     uniontypes, unsafe_convert, unwrap_unionall, unwrapva, vect, widen_diagonal,
     _uncompressed_ir, maybe_add_binding_backedge!, datatype_min_ninitialized,
-    partialstruct_undef_length, partialstruct_init_undef
+    partialstruct_init_undefs, fieldcount_noerror
 using Base.Order
 
 import Base: ==, _topmod, append!, convert, copy, copy!, findall, first, get, get!,
@@ -192,23 +193,22 @@ end
 module IRShow end # relies on string and IO operations defined in Base
 baremodule TrimVerifier end # relies on IRShow, so define this afterwards
 
-function load_irshow!()
-    if isdefined(Base, :end_base_include)
-        # This code path is exclusively for Revise, which may want to re-run this
-        # after bootstrap.
-        Compilerdir = Base.dirname(Base.String(@__SOURCE_FILE__))
-        include(IRShow, Base.joinpath(Compilerdir, "ssair/show.jl"))
-        include(TrimVerifier, Base.joinpath(Compilerdir, "verifytrim.jl"))
-    else
+if isdefined(Base, :end_base_include)
+    # When this module is loaded as the standard library, include these files as usual
+    include(IRShow, "ssair/show.jl")
+    include(TrimVerifier, "verifytrim.jl")
+else
+    function load_irshow!()
+        Base.delete_method(Base.which(verify_typeinf_trim, (IO, Vector{Any}, Bool)),)
         include(IRShow, "ssair/show.jl")
         include(TrimVerifier, "verifytrim.jl")
     end
-end
-if !isdefined(Base, :end_base_include)
-    # During bootstrap, skip including this file and defer it to base/show.jl to include later
-else
-    # When this module is loaded as the standard library, include this file as usual
-    load_irshow!()
+    function verify_typeinf_trim(io::IO, codeinfos::Vector{Any}, onlywarn::Bool)
+        # stub implementation
+        msg = "--trim verifier not defined"
+        onlywarn ? println(io, msg) : error(msg)
+    end
+    # During bootstrap, skip including these files and defer to base/show.jl to include it later
 end
 
 end # baremodule Compiler