fix accumulated bugs with stdlib initialization (#26241)

These were making it hard to turn off certain stdlibs,
which in turn made it hard to test alternate configurations and compiler changes.
And some environment variables weren't being handled well (or tested).

Author: vtjnash

base/client.jl

@@ -298,11 +298,6 @@ function exec_options(opts)
     # load ~/.julia/config/startup.jl file
     startup && load_julia_startup()
 
-    if repl || is_interactive
-        # load interactive-only libraries
-        eval(Main, :(using InteractiveUtils))
-    end
-
     # process cmds list
     for (cmd, arg) in cmds
         if cmd == 'e'
@@ -387,6 +382,18 @@ const REPL_MODULE_REF = Ref{Module}()
 # run the requested sort of evaluation loop on stdio
 function run_main_repl(interactive::Bool, quiet::Bool, banner::Bool, history_file::Bool, color_set::Bool)
     global active_repl
+    # load interactive-only libraries
+    if !isdefined(Main, :InteractiveUtils)
+        try
+            let InteractiveUtils = require(PkgId(UUID(0xb77e0a4c_d291_57a0_90e8_8db25a27a240), "InteractiveUtils"))
+                eval(Main, :(const InteractiveUtils = $InteractiveUtils))
+                eval(Main, :(using .InteractiveUtils))
+            end
+        catch ex
+            @warn "Failed to insert InteractiveUtils into module Main" exception=(ex, catch_backtrace())
+        end
+    end
+
     if interactive && isassigned(REPL_MODULE_REF)
         invokelatest(REPL_MODULE_REF[]) do REPL
             term_env = get(ENV, "TERM", @static Sys.iswindows() ? "" : "dumb")

base/initdefs.jl

@@ -46,7 +46,7 @@ const DEPOT_PATH = String[]
 function init_depot_path(BINDIR = Sys.BINDIR)
     if haskey(ENV, "JULIA_DEPOT_PATH")
         depots = split(ENV["JULIA_DEPOT_PATH"], Sys.iswindows() ? ';' : ':')
-        push!(empty!(DEPOT_PATH), map(expanduser, depots))
+        append!(empty!(DEPOT_PATH), map(expanduser, depots))
     else
         push!(DEPOT_PATH, joinpath(homedir(), ".julia"))
     end
@@ -80,6 +80,7 @@ function show(io::IO, env::NamedEnv)
 end
 
 function parse_env(env::Union{String,SubString{String}})
+    isempty(env) && return Any[]
     env == "@" && return CurrentEnv()
     env == "@!" && return CurrentEnv(create=true)
     if env[1] == '@'
@@ -106,7 +107,7 @@ function parse_load_path(str::String)
     envs = Any[split(str, Sys.iswindows() ? ';' : ':');]
     for (i, env) in enumerate(envs)
         if '|' in env
-            envs[i] = [parse_env(e) for e in split(env, '|')]
+            envs[i] = Any[parse_env(e) for e in split(env, '|')]
         else
             envs[i] = parse_env(env)
         end
@@ -122,17 +123,6 @@ function init_load_path(BINDIR = Sys.BINDIR)
     push!(LOAD_PATH, abspath(BINDIR, "..", "share", "julia", "site", vers))
 end
 
-function early_init()
-    Sys._early_init()
-    # make sure OpenBLAS does not set CPU affinity (#1070, #9639)
-    ENV["OPENBLAS_MAIN_FREE"] = get(ENV, "OPENBLAS_MAIN_FREE",
-                                get(ENV, "GOTOBLAS_MAIN_FREE", "1"))
-    if Sys.CPU_CORES > 8 && !("OPENBLAS_NUM_THREADS" in keys(ENV)) && !("OMP_NUM_THREADS" in keys(ENV))
-        # Prevent openblas from starting too many threads, unless/until specifically requested
-        ENV["OPENBLAS_NUM_THREADS"] = 8
-    end
-end
-
 const atexit_hooks = []
 
 """

base/logging.jl

@@ -367,8 +367,6 @@ end
 
 LogState(logger) = LogState(LogLevel(min_enabled_level(logger)), logger)
 
-_global_logstate = LogState(NullLogger())
-
 function current_logstate()
     logstate = current_task().logstate
     (logstate != nothing ? logstate : _global_logstate)::LogState
@@ -493,4 +491,6 @@ function handle_message(logger::SimpleLogger, level, message, _module, group, id
     nothing
 end
 
+_global_logstate = LogState(SimpleLogger(Core.stderr, CoreLogging.Info))
+
 end # CoreLogging

base/sysimg.jl

@@ -324,7 +324,6 @@ include("weakkeydict.jl")
 # Logging
 include("logging.jl")
 using .CoreLogging
-global_logger(SimpleLogger(Core.stderr, CoreLogging.Info))
 
 # To limit dependency on rand functionality (implemented in the Random
 # module), Crand is used in file.jl, and could be used in error.jl
@@ -343,7 +342,7 @@ Crand(::Type{Float64}) = Crand(UInt32) / 2^32
 """
     Csrand([seed])
 
-Interface the the C `srand(seed)` function.
+Interface with the C `srand(seed)` function.
 """
 Csrand(seed=floor(time())) = ccall(:srand, Cvoid, (Cuint,), seed)
 
@@ -456,7 +455,7 @@ include("loading.jl")
 include("util.jl")
 
 # set up depot & load paths to be able to find stdlib packages
-let BINDIR = ccall(:jl_get_julia_bindir, Any, ())
+let BINDIR = Sys.BINDIR
     init_depot_path(BINDIR)
     init_load_path(BINDIR)
 end
@@ -483,17 +482,28 @@ end_base_include = time_ns()
 
 if is_primary_base_module
 function __init__()
+    # try to ensuremake sure OpenBLAS does not set CPU affinity (#1070, #9639)
+    if !haskey(ENV, "OPENBLAS_MAIN_FREE") && !haskey(ENV, "GOTOBLAS_MAIN_FREE")
+        ENV["OPENBLAS_MAIN_FREE"] = "1"
+    end
+    # And try to prevent openblas from starting too many threads, unless/until specifically requested
+    if !haskey(ENV, "OPENBLAS_NUM_THREADS") && !haskey(ENV, "OMP_NUM_THREADS")
+        cpu_cores = Sys.CPU_CORES::Int
+        if cpu_cores > 8 # always at most 8
+            ENV["OPENBLAS_NUM_THREADS"] = "8"
+        elseif haskey(ENV, "JULIA_CPU_CORES") # or exactly as specified
+            ENV["OPENBLAS_NUM_THREADS"] = cpu_cores
+        end # otherwise, trust that openblas will pick CPU_CORES anyways, without any intervention
+    end
     # for the few uses of Crand in Base:
     Csrand()
     # Base library init
     reinit_stdio()
-    Logging = root_module(PkgId(UUID(0x56ddb016_857b_54e1_b83d_db4d58db5568), "Logging"))
-    global_logger(Logging.ConsoleLogger(stderr))
     Multimedia.reinit_displays() # since Multimedia.displays uses stdout as fallback
-    early_init()
+    # initialize loading
     init_depot_path()
     init_load_path()
-    init_threadcall()
+    nothing
 end
 
 INCLUDE_STATE = 3 # include = include_relative
@@ -505,7 +515,6 @@ end # baremodule Base
 
 using .Base
 
-
 # Ensure this file is also tracked
 pushfirst!(Base._included_files, (@__MODULE__, joinpath(@__DIR__, "sysimg.jl")))
 
@@ -915,8 +924,12 @@ end
 end
 end
 
+# Clear global state
+empty!(Core.ARGS)
+empty!(Base.ARGS)
 empty!(DEPOT_PATH)
 empty!(LOAD_PATH)
+@eval Base.Sys BINDIR = ""
 
 let
 tot_time_userimg = @elapsed (Base.isfile("userimg.jl") && Base.include(Main, "userimg.jl"))

base/sysinfo.jl

@@ -27,14 +27,13 @@ export BINDIR,
 
 import ..Base: show
 
+global BINDIR = ccall(:jl_get_julia_bindir, Any, ())
 """
     Sys.BINDIR
 
 A string containing the full path to the directory containing the `julia` executable.
 """
-BINDIR = ccall(:jl_get_julia_bindir, Any, ())
-
-_early_init() = global BINDIR = ccall(:jl_get_julia_bindir, Any, ())
+:BINDIR
 
 # helper to avoid triggering precompile warnings
 
@@ -78,12 +77,21 @@ Standard word size on the current machine, in bits.
 const WORD_SIZE = Core.sizeof(Int) * 8
 
 function __init__()
-    global CPU_CORES =
-        haskey(ENV,"JULIA_CPU_CORES") ? parse(Int,ENV["JULIA_CPU_CORES"]) :
-                                        Int(ccall(:jl_cpu_cores, Int32, ()))
+    env_cores = get(ENV, "JULIA_CPU_CORES", "")
+    global CPU_CORES = if !isempty(env_cores)
+        env_cores = tryparse(Int, env_cores)
+        if !(env_cores isa Int && env_cores > 0)
+            Core.print(Core.stderr, "WARNING: couldn't parse `JULIA_CPU_CORES` environment variable. Defaulting Sys.CPU_CORES to 1.\n")
+            env_cores = 1
+        end
+        env_cores
+    else
+        Int(ccall(:jl_cpu_cores, Int32, ()))
+    end
     global SC_CLK_TCK = ccall(:jl_SC_CLK_TCK, Clong, ())
     global CPU_NAME = ccall(:jl_get_cpu_name, Ref{String}, ())
     global JIT = ccall(:jl_get_JIT, Ref{String}, ())
+    global BINDIR = ccall(:jl_get_julia_bindir, Any, ())
 end
 
 mutable struct UV_cpu_info_t

base/threadcall.jl

@@ -10,10 +10,6 @@ function notify_fun(idx)
     return
 end
 
-function init_threadcall()
-    global c_notify_fun = cfunction(notify_fun, Cvoid, Tuple{Cint})
-end
-
 """
     @threadcall((cfunc, clib), rettype, (argtypes...), argvals...)
 
@@ -65,6 +61,7 @@ end
 function do_threadcall(wrapper::Function, rettype::Type, argtypes::Vector, argvals::Vector)
     # generate function pointer
     fun_ptr = cfunction(wrapper, Int, Tuple{Ptr{Cvoid}, Ptr{Cvoid}})
+    c_notify_fun = cfunction(notify_fun, Cvoid, Tuple{Cint})
 
     # cconvert, root and unsafe_convert arguments
     roots = Any[]

stdlib/Distributed/test/distributed_exec.jl

@@ -948,7 +948,7 @@ const get_num_threads = function() # anonymous so it will be serialized when cal
 
         # OSX BLAS looks at an environment variable
         if Sys.isapple()
-            return ENV["VECLIB_MAXIMUM_THREADS"]
+            return tryparse(Cint, get(ENV, "VECLIB_MAXIMUM_THREADS", "1"))
         end
     end
 
@@ -969,11 +969,12 @@ function test_blas_config(pid, expected)
 end
 
 function test_add_procs_threaded_blas()
-    if get_num_threads() === nothing
+    master_blas_thread_count = get_num_threads()
+    if master_blas_thread_count === nothing
         @warn "Skipping blas num threads tests due to unsupported blas version"
         return
     end
-    master_blas_thread_count = get_num_threads()
+    @test master_blas_thread_count <= 8 # check that Base set the environment variable in __init__ before LinearAlgebra dlopen'd it
 
     # Test with default enable_threaded_blas false
     processes_added = addprocs_with_testenv(2)

stdlib/Logging/src/Logging.jl

@@ -52,4 +52,8 @@ include("ConsoleLogger.jl")
 # 2. AbstractLogger message related functions:
 #  handle_message, shouldlog, min_enabled_level, catch_exceptions,
 
+function __init__()
+    global_logger(ConsoleLogger(stderr))
+end
+
 end

test/cmdlineargs.jl

@@ -10,7 +10,7 @@ function writereadpipeline(input, exename)
         write(p.in, input)
         close(p.in)
     end
-    return read(p.out, String)
+    return (read(p.out, String), success(p))
 end
 
 # helper function for returning stderr and stdout
@@ -25,6 +25,52 @@ function readchomperrors(exename::Cmd)
 end
 
 
+let exename = `$(Base.julia_cmd()) --sysimage-native-code=yes --startup-file=no`
+    # tests for handling of ENV errors
+    let v = writereadpipeline("println(\"REPL: \", @which(less), @isdefined(InteractiveUtils))",
+                setenv(`$exename -i -E 'empty!(LOAD_PATH); @isdefined InteractiveUtils'`,
+                    "JULIA_LOAD_PATH"=>"", "JULIA_DEPOT_PATH"=>""))
+        @test v[1] == "false\nREPL: InteractiveUtilstrue\n"
+        @test v[2]
+    end
+    let v = writereadpipeline("println(\"REPL: \", InteractiveUtils)",
+                setenv(`$exename -i -e 'const InteractiveUtils = 3'`,
+                    "JULIA_LOAD_PATH"=>";;;:::", "JULIA_DEPOT_PATH"=>";;;:::"))
+        # TODO: ideally, `@which`, etc. would still work, but Julia can't handle `using $InterativeUtils`
+        @test v[1] == "REPL: 3\n"
+        @test v[2]
+    end
+    let v = readchomperrors(`$exename -i -e '
+            empty!(LOAD_PATH)
+            Base.unreference_module(Base.PkgId(Base.UUID(0xb77e0a4c_d291_57a0_90e8_8db25a27a240), "InteractiveUtils"))
+            '`)
+        # simulate not having a working version of InteractiveUtils,
+        # make sure this is a non-fatal error and the REPL still loads
+        @test v[1]
+        @test isempty(v[2])
+        @test startswith(v[3], """
+                         ┌ Warning: Failed to insert InteractiveUtils into module Main
+                         │   exception =
+                         │    ArgumentError: Module InteractiveUtils not found in current path.
+                         │    Run `Pkg.add("InteractiveUtils")` to install the InteractiveUtils package.
+                         │    Stacktrace:
+                         """)
+    end
+    for nc in ("0", "-2", "x", "2x", " ")
+        v = readchomperrors(setenv(`$exename -i -E 'Sys.CPU_CORES'`, "JULIA_CPU_CORES" => nc))
+        @test v[1]
+        @test v[2] == "1"
+        @test v[3] == "WARNING: couldn't parse `JULIA_CPU_CORES` environment variable. Defaulting Sys.CPU_CORES to 1."
+    end
+    real_cores = string(ccall(:jl_cpu_cores, Int32, ()))
+    for nc in ("1", " 1 ", " +1 ", " 0x1 ", "")
+        v = readchomperrors(setenv(`$exename -i -E 'Sys.CPU_CORES'`, "JULIA_CPU_CORES" => nc))
+        @test v[1]
+        @test v[2] == (isempty(nc) ? real_cores : "1")
+        @test isempty(v[3])
+    end
+end
+
 let exename = `$(Base.julia_cmd()) --sysimage-native-code=yes --startup-file=no`
     # --version
     let v = split(read(`$exename -v`, String), "julia version ")[end]
@@ -173,19 +219,25 @@ let exename = `$(Base.julia_cmd()) --sysimage-native-code=yes --startup-file=no`
 
     # -g
     @test readchomp(`$exename -E "Base.JLOptions().debug_level" -g`) == "2"
-    let code = read(`$exename -g0 -i -e "code_llvm(stdout, +, (Int64, Int64), false, true); exit()"`, String)
+    let code = writereadpipeline("code_llvm(stdout, +, (Int64, Int64), false, true)", `$exename -g0`)
+        @test code[2]
+        code = code[1]
         @test contains(code, "llvm.module.flags")
         @test !contains(code, "llvm.dbg.cu")
         @test !contains(code, "int.jl")
         @test !contains(code, "Int64")
     end
-    let code = read(`$exename -g1 -i -e "code_llvm(stdout, +, (Int64, Int64), false, true); exit()"`, String)
+    let code = writereadpipeline("code_llvm(stdout, +, (Int64, Int64), false, true)", `$exename -g1`)
+        @test code[2]
+        code = code[1]
         @test contains(code, "llvm.module.flags")
         @test contains(code, "llvm.dbg.cu")
         @test contains(code, "int.jl")
         @test !contains(code, "Int64")
     end
-    let code = read(`$exename -g2 -i -e "code_llvm(stdout, +, (Int64, Int64), false, true); exit()"`, String)
+    let code = writereadpipeline("code_llvm(stdout, +, (Int64, Int64), false, true)", `$exename -g2`)
+        @test code[2]
+        code = code[1]
         @test contains(code, "llvm.module.flags")
         @test contains(code, "llvm.dbg.cu")
         @test contains(code, "int.jl")
@@ -486,11 +538,11 @@ end
 
 # issue #6310
 let exename = `$(Base.julia_cmd()) --startup-file=no`
-    @test writereadpipeline("2+2", exename) == "4\n"
-    @test writereadpipeline("2+2\n3+3\n4+4", exename) == "4\n6\n8\n"
-    @test writereadpipeline("", exename) == ""
-    @test writereadpipeline("print(2)", exename) == "2"
-    @test writereadpipeline("print(2)\nprint(3)", exename) == "23"
+    @test writereadpipeline("2+2", exename) == ("4\n", true)
+    @test writereadpipeline("2+2\n3+3\n4+4", exename) == ("4\n6\n8\n", true)
+    @test writereadpipeline("", exename) == ("", true)
+    @test writereadpipeline("print(2)", exename) == ("2", true)
+    @test writereadpipeline("print(2)\nprint(3)", exename) == ("23", true)
     let infile = tempname()
         touch(infile)
         try