Merge branch 'master' into ib/repl_display_backend

Author: IanButterworth

HISTORY.md

@@ -52,6 +52,10 @@ Language changes
 * Errors during `getfield` now raise a new `FieldError` exception type instead of the generic
   `ErrorException` ([#54504]).
 * Macros in function-signature-position no longer require parentheses. E.g. `function @main(args) ... end` is now permitted, whereas `function (@main)(args) ... end` was required in prior Julia versions.
+* Calling `using` on a package name inside of that package of that name (especially relevant
+  for a submodule) now explicitly uses that package without examining the Manifest and
+  environment, which is identical to the behavior of `..Name`. This appears to better match
+  how users expect this to behave in the wild. ([#57727])
 
 Compiler/Runtime improvements
 -----------------------------

Makefile

@@ -110,13 +110,7 @@ julia-src-release julia-src-debug : julia-src-% : julia-deps julia_flisp.boot.in
 julia-cli-release julia-cli-debug: julia-cli-% : julia-deps
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/cli $*
 
-julia-sysimg-ji : $(TOP_LEVEL_PKG_LINK_TARGETS) julia-stdlib julia-base julia-cli-$(JULIA_BUILD_MODE) julia-src-$(JULIA_BUILD_MODE) | $(build_private_libdir)
-	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysimg-ji JULIA_EXECUTABLE='$(JULIA_EXECUTABLE)'
-
-julia-sysimg-bc : $(TOP_LEVEL_PKG_LINK_TARGETS) julia-stdlib julia-base julia-cli-$(JULIA_BUILD_MODE) julia-src-$(JULIA_BUILD_MODE) | $(build_private_libdir)
-	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysimg-bc JULIA_EXECUTABLE='$(JULIA_EXECUTABLE)'
-
-julia-sysimg-release julia-sysimg-debug : julia-sysimg-% : julia-sysimg-ji julia-src-%
+julia-sysimg-release julia-sysimg-debug : julia-sysimg-% : julia-src-% $(TOP_LEVEL_PKG_LINK_TARGETS) julia-stdlib julia-base julia-cli-% julia-src-% | $(build_private_libdir)
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysimg-$*
 
 julia-debug julia-release : julia-% : julia-sysimg-% julia-src-% julia-symlink julia-libccalltest \
@@ -177,6 +171,7 @@ release-candidate: release testall
 	@echo 14. Push to Juliaup (https://github.com/JuliaLang/juliaup/wiki/Adding-a-Julia-version)
 	@echo 15. Announce on mailing lists
 	@echo 16. Change master to release-0.X in base/version.jl and base/version_git.sh as in 4cb1e20
+	@echo 17. Move NEWS.md contents to HISTORY.md
 	@echo
 
 $(build_man1dir)/julia.1: $(JULIAHOME)/doc/man/julia.1 | $(build_man1dir)

base/Base.jl

@@ -213,9 +213,6 @@ using .PermutedDimsArrays
 include("sort.jl")
 using .Sort
 
-# BinaryPlatforms, used by Artifacts.  Needs `Sort`.
-include("binaryplatforms.jl")
-
 # Fast math
 include("fastmath.jl")
 using .FastMath
@@ -269,6 +266,9 @@ include("linking.jl")
 include("staticdata.jl")
 include("loading.jl")
 
+# BinaryPlatforms, used by Artifacts.  Needs `Sort`.
+include("binaryplatforms.jl")
+
 # misc useful functions & macros
 include("timing.jl")
 include("client.jl")

base/array.jl

@@ -1356,7 +1356,7 @@ end
 
 append!(a::AbstractVector, iter) = _append!(a, IteratorSize(iter), iter)
 push!(a::AbstractVector, iter...) = append!(a, iter)
-append!(a::AbstractVector, iter...) = (for v in iter; append!(a, v); end; return a)
+append!(a::AbstractVector, iter...) = (foreach(v -> append!(a, v), iter); a)
 
 function _append!(a::AbstractVector, ::Union{HasLength,HasShape}, iter)
     n = Int(length(iter))::Int

base/binaryplatforms.jl

@@ -841,20 +841,48 @@ function parse_dl_name_version(path::AbstractString, os::AbstractString)
     return parse_dl_name_version(string(path)::String, string(os)::String)
 end
 
+function get_csl_member(member::Symbol)
+    # If CompilerSupportLibraries_jll is an stdlib, we can just grab things from it
+    csl_pkgids = filter(pkgid -> pkgid.name == "CompilerSupportLibraries_jll", keys(Base.loaded_modules))
+    if !isempty(csl_pkgids)
+        CSL_mod = Base.loaded_modules[first(csl_pkgids)]
+
+        # This can fail during bootstrap, so we skip in that case.
+        if isdefined(CSL_mod, member)
+            return getproperty(CSL_mod, member)
+        end
+    end
+
+    return nothing
+end
+
 """
     detect_libgfortran_version()
 
 Inspects the current Julia process to determine the libgfortran version this Julia is
-linked against (if any).
+linked against (if any).  Returns `nothing` if no libgfortran version dependence is
+detected.
 """
 function detect_libgfortran_version()
-    libgfortran_paths = filter!(x -> occursin("libgfortran", x), Libdl.dllist())
-    if isempty(libgfortran_paths)
+    function get_libgfortran_path()
+        # If CompilerSupportLibraries_jll is an stdlib, we can just directly ask for
+        # the path here, without checking `dllist()`:
+        libgfortran_path = get_csl_member(:libgfortran_path)
+        if libgfortran_path !== nothing
+            return libgfortran_path::String
+        end
+
+        # Otherwise, look for it having already been loaded by something
+        libgfortran_paths = filter!(x -> occursin("libgfortran", x), Libdl.dllist())
+        if !isempty(libgfortran_paths)
+            return first(libgfortran_paths)::String
+        end
+
         # One day, I hope to not be linking against libgfortran in base Julia
         return nothing
     end
-    libgfortran_path = first(libgfortran_paths)
 
+    libgfortran_path = get_libgfortran_path()
     name, version = parse_dl_name_version(libgfortran_path, os())
     if version === nothing
         # Even though we complain about this, we allow it to continue in the hopes that
@@ -878,24 +906,37 @@ it is linked against (if any).  `max_minor_version` is the latest version in the
 3.4 series of GLIBCXX where the search is performed.
 """
 function detect_libstdcxx_version(max_minor_version::Int=30)
-    libstdcxx_paths = filter!(x -> occursin("libstdc++", x), Libdl.dllist())
-    if isempty(libstdcxx_paths)
-        # This can happen if we were built by clang, so we don't link against
-        # libstdc++ at all.
+    function get_libstdcxx_handle()
+        # If CompilerSupportLibraries_jll is an stdlib, we can just directly open it
+        libstdcxx = get_csl_member(:libstdcxx)
+        if libstdcxx !== nothing
+            return nothing
+        end
+
+        # Otherwise, look for it having already been loaded by something
+        libstdcxx_paths = filter!(x -> occursin("libstdc++", x), Libdl.dllist())
+        if !isempty(libstdcxx_paths)
+            return Libdl.dlopen(first(libstdcxx_paths), Libdl.RTLD_NOLOAD)::Ptr{Cvoid}
+        end
+
+        # One day, I hope to not be linking against libgfortran in base Julia
         return nothing
     end
 
     # Brute-force our way through GLIBCXX_* symbols to discover which version we're linked against
-    hdl = Libdl.dlopen(first(libstdcxx_paths))::Ptr{Cvoid}
-    # Try all GLIBCXX versions down to GCC v4.8:
-    # https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html
-    for minor_version in max_minor_version:-1:18
-        if Libdl.dlsym(hdl, "GLIBCXX_3.4.$(minor_version)"; throw_error=false) !== nothing
-            Libdl.dlclose(hdl)
-            return VersionNumber("3.4.$(minor_version)")
+    libstdcxx = get_libstdcxx_handle()
+
+    if libstdcxx !== nothing
+        # Try all GLIBCXX versions down to GCC v4.8:
+        # https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html
+        for minor_version in max_minor_version:-1:18
+            if Libdl.dlsym(libstdcxx, "GLIBCXX_3.4.$(minor_version)"; throw_error=false) !== nothing
+                Libdl.dlclose(libstdcxx)
+                return VersionNumber("3.4.$(minor_version)")
+            end
         end
     end
-    Libdl.dlclose(hdl)
+    Libdl.dlclose(libstdcxx)
     return nothing
 end
 

base/libdl.jl

@@ -333,14 +333,17 @@ struct LazyLibraryPath
     pieces::Tuple{Vararg{Any}}
     LazyLibraryPath(pieces...) = new(pieces)
 end
-Base.string(llp::LazyLibraryPath) = joinpath(String[string(p) for p in llp.pieces])
+@inline Base.string(llp::LazyLibraryPath) = joinpath(String[string(p) for p in llp.pieces])
 Base.cconvert(::Type{Cstring}, llp::LazyLibraryPath) = Base.cconvert(Cstring, string(llp))
 # Define `print` so that we can wrap this in a `LazyString`
 Base.print(io::IO, llp::LazyLibraryPath) = print(io, string(llp))
 
-# Helper to get `Sys.BINDIR` at runtime
-struct SysBindirGetter; end
-Base.string(::SysBindirGetter) = dirname(Sys.BINDIR)
+# Helper to get `$(private_shlibdir)` at runtime
+struct PrivateShlibdirGetter; end
+const private_shlibdir = Base.OncePerProcess{String}() do
+    dirname(dlpath("libjulia-internal"))
+end
+Base.string(::PrivateShlibdirGetter) = private_shlibdir()
 
 """
     BundledLazyLibraryPath
@@ -349,10 +352,10 @@ Helper type for lazily constructed library paths that are stored within the
 bundled Julia distribution, primarily for use by Base modules.
 
 ```
-libfoo = LazyLibrary(BundledLazyLibraryPath("lib/libfoo.so.1.2.3"))
+libfoo = LazyLibrary(BundledLazyLibraryPath("libfoo.so.1.2.3"))
 ```
 """
-BundledLazyLibraryPath(subpath) = LazyLibraryPath(SysBindirGetter(), subpath)
+BundledLazyLibraryPath(subpath) = LazyLibraryPath(PrivateShlibdirGetter(), subpath)
 
 
 """

base/loading.jl

@@ -1280,17 +1280,21 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No
         sv = try
             if ocachepath !== nothing
                 @debug "Loading object cache file $ocachepath for $(repr("text/plain", pkg))"
-                ccall(:jl_restore_package_image_from_file, Ref{SimpleVector}, (Cstring, Any, Cint, Cstring, Cint),
+                ccall(:jl_restore_package_image_from_file, Any, (Cstring, Any, Cint, Cstring, Cint),
                     ocachepath, depmods, #=completeinfo=#false, pkg.name, ignore_native)
             else
                 @debug "Loading cache file $path for $(repr("text/plain", pkg))"
-                ccall(:jl_restore_incremental, Ref{SimpleVector}, (Cstring, Any, Cint, Cstring),
+                ccall(:jl_restore_incremental, Any, (Cstring, Any, Cint, Cstring),
                     path, depmods, #=completeinfo=#false, pkg.name)
             end
         finally
             lock(require_lock)
         end
+        if isa(sv, Exception)
+            return sv
+        end
 
+        sv = sv::SimpleVector
         edges = sv[3]::Vector{Any}
         ext_edges = sv[4]::Union{Nothing,Vector{Any}}
         extext_methods = sv[5]::Vector{Any}
@@ -1438,7 +1442,6 @@ function run_module_init(mod::Module, i::Int=1)
 end
 
 function run_package_callbacks(modkey::PkgId)
-    @assert modkey != precompilation_target
     run_extension_callbacks(modkey)
     assert_havelock(require_lock)
     unlock(require_lock)
@@ -1568,7 +1571,7 @@ function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any}
             uuid_trigger = UUID(totaldeps[trigger]::String)
             trigger_id = PkgId(uuid_trigger, trigger)
             push!(trigger_ids, trigger_id)
-            if !haskey(Base.loaded_modules, trigger_id) || haskey(package_locks, trigger_id) || (trigger_id == precompilation_target)
+            if !haskey(Base.loaded_modules, trigger_id) || haskey(package_locks, trigger_id)
                 trigger1 = get!(Vector{ExtensionId}, EXT_DORMITORY, trigger_id)
                 push!(trigger1, gid)
             else
@@ -1581,7 +1584,6 @@ end
 loading_extension::Bool = false
 loadable_extensions::Union{Nothing,Vector{PkgId}} = nothing
 precompiling_extension::Bool = false
-precompilation_target::Union{Nothing,PkgId} = nothing
 function run_extension_callbacks(extid::ExtensionId)
     assert_havelock(require_lock)
     succeeded = try
@@ -2178,7 +2180,6 @@ function canstart_loading(modkey::PkgId, build_id::UInt128, stalecheck::Bool)
     # load already in progress for this module on the task
     task, cond = loading
     deps = String[modkey.name]
-    pkgid = modkey
     assert_havelock(cond.lock)
     if debug_loading_deadlocks && current_task() !== task
         waiters = Dict{Task,Pair{Task,PkgId}}() # invert to track waiting tasks => loading tasks
@@ -2198,18 +2199,26 @@ function canstart_loading(modkey::PkgId, build_id::UInt128, stalecheck::Bool)
         end
     end
     if current_task() === task
-        others = String[modkey.name] # repeat this to emphasize the cycle here
+        push!(deps, modkey.name) # repeat this to emphasize the cycle here
+        others = Set{String}()
         for each in package_locks # list the rest of the packages being loaded too
             if each[2][1] === task
                 other = each[1].name
-                other == modkey.name || other == pkgid.name || push!(others, other)
+                other == modkey.name || push!(others, other)
             end
         end
+        # remove duplicates from others already in deps
+        for dep in deps
+            delete!(others, dep)
+        end
         msg = sprint(deps, others) do io, deps, others
             print(io, "deadlock detected in loading ")
-            join(io, deps, " -> ")
-            print(io, " -> ")
-            join(io, others, " && ")
+            join(io, deps, " using ")
+            if !isempty(others)
+                print(io, " (while loading ")
+                join(io, others, " and ")
+                print(io, ")")
+            end
         end
         throw(ConcurrencyViolationError(msg))
     end
@@ -2383,6 +2392,10 @@ function __require(into::Module, mod::Symbol)
         error("`using/import $mod` outside of a Module detected. Importing a package outside of a module \
          is not allowed during package precompilation.")
     end
+    topmod = moduleroot(into)
+    if nameof(topmod) === mod
+        return topmod
+    end
     @lock require_lock begin
     LOADING_CACHE[] = LoadingCache()
     try
@@ -2491,10 +2504,7 @@ function _require_prelocked(uuidkey::PkgId, env=nothing)
         try
             toplevel_load[] = false
             m = __require_prelocked(uuidkey, env)
-            if m === nothing
-                error("package `$(uuidkey.name)` did not define the expected \
-                      module `$(uuidkey.name)`, check for typos in package module name")
-            end
+            m isa Module || check_package_module_loaded_error(uuidkey)
         finally
             toplevel_load[] = last
             end_loading(uuidkey, m)
@@ -2984,6 +2994,9 @@ const newly_inferred = CodeInstance[]
 function include_package_for_output(pkg::PkgId, input::String, depot_path::Vector{String}, dl_load_path::Vector{String}, load_path::Vector{String},
                                     concrete_deps::typeof(_concrete_dependencies), source::Union{Nothing,String})
 
+    @lock require_lock begin
+    m = start_loading(pkg, UInt128(0), false)
+    @assert m === nothing
     append!(empty!(Base.DEPOT_PATH), depot_path)
     append!(empty!(Base.DL_LOAD_PATH), dl_load_path)
     append!(empty!(Base.LOAD_PATH), load_path)
@@ -2992,6 +3005,8 @@ function include_package_for_output(pkg::PkgId, input::String, depot_path::Vecto
     Base._track_dependencies[] = true
     get!(Base.PkgOrigin, Base.pkgorigins, pkg).path = input
     append!(empty!(Base._concrete_dependencies), concrete_deps)
+    end
+
     uuid_tuple = pkg.uuid === nothing ? (UInt64(0), UInt64(0)) : convert(NTuple{2, UInt64}, pkg.uuid)
 
     ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, uuid_tuple)
@@ -3010,21 +3025,22 @@ function include_package_for_output(pkg::PkgId, input::String, depot_path::Vecto
         ccall(:jl_set_newly_inferred, Cvoid, (Any,), nothing)
     end
     # check that the package defined the expected module so we can give a nice error message if not
-    Base.check_package_module_loaded(pkg)
+    m = maybe_root_module(pkg)
+    m isa Module || check_package_module_loaded_error(pkg)
 
     # Re-populate the runtime's newly-inferred array, which will be included
     # in the output. We removed it above to avoid including any code we may
     # have compiled for error handling and validation.
     ccall(:jl_set_newly_inferred, Cvoid, (Any,), newly_inferred)
+    @lock require_lock end_loading(pkg, m)
+    # insert_extension_triggers(pkg)
+    # run_package_callbacks(pkg)
 end
 
-function check_package_module_loaded(pkg::PkgId)
-    if !haskey(Base.loaded_modules, pkg)
-        # match compilecache error type for non-125 errors
-        error("$(repr("text/plain", pkg)) did not define the expected module `$(pkg.name)`, \
-            check for typos in package module name")
-    end
-    return nothing
+function check_package_module_loaded_error(pkg)
+    # match compilecache error type for non-125 errors
+    error("package `$(pkg.name)` did not define the expected \
+          module `$(pkg.name)`, check for typos in package module name")
 end
 
 # protects against PkgId and UUID being imported and losing Base prefix
@@ -3100,7 +3116,6 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o::
         Base.track_nested_precomp($(_pkg_str(vcat(Base.precompilation_stack, pkg))))
         Base.loadable_extensions = $(_pkg_str(loadable_exts))
         Base.precompiling_extension = $(loading_extension)
-        Base.precompilation_target = $(_pkg_str(pkg))
         Base.include_package_for_output($(_pkg_str(pkg)), $(repr(abspath(input))), $(repr(depot_path)), $(repr(dl_load_path)),
             $(repr(load_path)), $(_pkg_str(concrete_deps)), $(repr(source_path(nothing))))
         """)

base/multidimensional.jl

@@ -91,7 +91,11 @@ module IteratorsMD
     flatten(I::Tuple{Any}) = Tuple(I[1])
     @inline flatten(I::Tuple) = (Tuple(I[1])..., flatten(tail(I))...)
     CartesianIndex(index::Tuple{Vararg{Union{Integer, CartesianIndex}}}) = CartesianIndex(index...)
-    show(io::IO, i::CartesianIndex) = (print(io, "CartesianIndex"); show(io, i.I))
+    function show(io::IO, i::CartesianIndex)
+        print(io, "CartesianIndex(")
+        join(io, i.I, ", ")
+        print(io, ")")
+    end
 
     # length
     length(::CartesianIndex{N}) where {N} = N

base/toml_parser.jl

@@ -770,7 +770,7 @@ isvalid_hex(c::Char) = isdigit(c) || ('a' <= c <= 'f') || ('A' <= c <= 'F')
 isvalid_oct(c::Char) = '0' <= c <= '7'
 isvalid_binary(c::Char) = '0' <= c <= '1'
 
-const ValidSigs = Union{typeof.([isvalid_hex, isvalid_oct, isvalid_binary, isdigit])...}
+const ValidSigs = Union{typeof(isvalid_hex), typeof(isvalid_oct), typeof(isvalid_binary), typeof(isdigit)}
 # This function eats things accepted by `f` but also allows eating `_` in between
 # digits. Returns if it ate at lest one character and if it ate an underscore
 function accept_batch_underscore(l::Parser, f::ValidSigs, fail_if_underscore=true)::Err{Tuple{Bool, Bool}}