Skip to content

Commit 5b677a1

Browse files
authored
Revert "Extensions: make loading of extensions independent of what packages are in the sysimage (#52841) (#56234)
This reverts commit 08d229f. There are some bugs now where extensions do not load when their package has been put into the sysimage. #52841 was made because it was common to get cycles otherwise but with #55589 that should be much less of a problem. Subsumes #54750.
2 parents 31f7df6 + 319ee70 commit 5b677a1

File tree

6 files changed

+24
-57
lines changed

6 files changed

+24
-57
lines changed

base/Base.jl

-1
Original file line numberDiff line numberDiff line change
@@ -646,7 +646,6 @@ function __init__()
646646
init_load_path()
647647
init_active_project()
648648
append!(empty!(_sysimage_modules), keys(loaded_modules))
649-
empty!(explicit_loaded_modules)
650649
empty!(loaded_precompiles) # If we load a packageimage when building the image this might not be empty
651650
for (mod, key) in module_keys
652651
push!(get!(Vector{Module}, loaded_precompiles, key), mod)

base/loading.jl

+23-26
Original file line numberDiff line numberDiff line change
@@ -974,14 +974,14 @@ function explicit_manifest_deps_get(project_file::String, where::PkgId, name::St
974974
entry = entry::Dict{String, Any}
975975
uuid = get(entry, "uuid", nothing)::Union{String, Nothing}
976976
uuid === nothing && continue
977+
# deps is either a list of names (deps = ["DepA", "DepB"]) or
978+
# a table of entries (deps = {"DepA" = "6ea...", "DepB" = "55d..."}
979+
deps = get(entry, "deps", nothing)::Union{Vector{String}, Dict{String, Any}, Nothing}
977980
if UUID(uuid) === where.uuid
978981
found_where = true
979-
# deps is either a list of names (deps = ["DepA", "DepB"]) or
980-
# a table of entries (deps = {"DepA" = "6ea...", "DepB" = "55d..."}
981-
deps = get(entry, "deps", nothing)::Union{Vector{String}, Dict{String, Any}, Nothing}
982982
if deps isa Vector{String}
983983
found_name = name in deps
984-
break
984+
found_name && @goto done
985985
elseif deps isa Dict{String, Any}
986986
deps = deps::Dict{String, Any}
987987
for (dep, uuid) in deps
@@ -1000,30 +1000,33 @@ function explicit_manifest_deps_get(project_file::String, where::PkgId, name::St
10001000
return PkgId(UUID(uuid), name)
10011001
end
10021002
exts = extensions[where.name]::Union{String, Vector{String}}
1003+
weakdeps = get(entry, "weakdeps", nothing)::Union{Vector{String}, Dict{String, Any}, Nothing}
10031004
if (exts isa String && name == exts) || (exts isa Vector{String} && name in exts)
1004-
weakdeps = get(entry, "weakdeps", nothing)::Union{Vector{String}, Dict{String, Any}, Nothing}
1005-
if weakdeps !== nothing
1006-
if weakdeps isa Vector{String}
1007-
found_name = name in weakdeps
1008-
break
1009-
elseif weakdeps isa Dict{String, Any}
1010-
weakdeps = weakdeps::Dict{String, Any}
1011-
for (dep, uuid) in weakdeps
1012-
uuid::String
1013-
if dep === name
1014-
return PkgId(UUID(uuid), name)
1005+
for deps′ in [weakdeps, deps]
1006+
if deps′ !== nothing
1007+
if deps′ isa Vector{String}
1008+
found_name = name in deps′
1009+
found_name && @goto done
1010+
elseif deps′ isa Dict{String, Any}
1011+
deps′ = deps′::Dict{String, Any}
1012+
for (dep, uuid) in deps′
1013+
uuid::String
1014+
if dep === name
1015+
return PkgId(UUID(uuid), name)
1016+
end
1017+
end
10151018
end
10161019
end
10171020
end
10181021
end
1019-
end
10201022
# `name` is not an ext, do standard lookup as if this was the parent
10211023
return identify_package(PkgId(UUID(uuid), dep_name), name)
10221024
end
10231025
end
10241026
end
10251027
end
10261028
end
1029+
@label done
10271030
found_where || return nothing
10281031
found_name || return PkgId(name)
10291032
# Only reach here if deps was not a dict which mean we have a unique name for the dep
@@ -1554,7 +1557,7 @@ function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any}
15541557
# TODO: Better error message if this lookup fails?
15551558
uuid_trigger = UUID(totaldeps[trigger]::String)
15561559
trigger_id = PkgId(uuid_trigger, trigger)
1557-
if !haskey(explicit_loaded_modules, trigger_id) || haskey(package_locks, trigger_id)
1560+
if !haskey(Base.loaded_modules, trigger_id) || haskey(package_locks, trigger_id)
15581561
trigger1 = get!(Vector{ExtensionId}, EXT_DORMITORY, trigger_id)
15591562
push!(trigger1, gid)
15601563
else
@@ -2428,9 +2431,8 @@ function __require_prelocked(uuidkey::PkgId, env=nothing)
24282431
insert_extension_triggers(uuidkey)
24292432
# After successfully loading, notify downstream consumers
24302433
run_package_callbacks(uuidkey)
2431-
elseif !haskey(explicit_loaded_modules, uuidkey)
2432-
explicit_loaded_modules[uuidkey] = m
2433-
run_package_callbacks(uuidkey)
2434+
else
2435+
newm = root_module(uuidkey)
24342436
end
24352437
return m
24362438
end
@@ -2443,7 +2445,6 @@ end
24432445
PkgOrigin() = PkgOrigin(nothing, nothing, nothing)
24442446
const pkgorigins = Dict{PkgId,PkgOrigin}()
24452447

2446-
const explicit_loaded_modules = Dict{PkgId,Module}() # Emptied on Julia start
24472448
const loaded_modules = Dict{PkgId,Module}() # available to be explicitly loaded
24482449
const loaded_precompiles = Dict{PkgId,Vector{Module}}() # extended (complete) list of modules, available to be loaded
24492450
const loaded_modules_order = Vector{Module}()
@@ -2483,7 +2484,6 @@ end
24832484
end
24842485
maybe_loaded_precompile(key, module_build_id(m)) === nothing && push!(loaded_modules_order, m)
24852486
loaded_modules[key] = m
2486-
explicit_loaded_modules[key] = m
24872487
module_keys[m] = key
24882488
end
24892489
nothing
@@ -2515,9 +2515,6 @@ loaded_modules_array() = @lock require_lock copy(loaded_modules_order)
25152515
# after unreference_module, a subsequent require call will try to load a new copy of it, if stale
25162516
# reload(m) = (unreference_module(m); require(m))
25172517
function unreference_module(key::PkgId)
2518-
if haskey(explicit_loaded_modules, key)
2519-
m = pop!(explicit_loaded_modules, key)
2520-
end
25212518
if haskey(loaded_modules, key)
25222519
m = pop!(loaded_modules, key)
25232520
# need to ensure all modules are GC rooted; will still be referenced
@@ -3126,7 +3123,7 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in
31263123
# build up the list of modules that we want the precompile process to preserve
31273124
if keep_loaded_modules
31283125
concrete_deps = copy(_concrete_dependencies)
3129-
for (pkgreq, modreq) in loaded_modules # TODO: convert all relevant staleness heuristics to use explicit_loaded_modules instead
3126+
for (pkgreq, modreq) in loaded_modules
31303127
if !(pkgreq === Main || pkgreq === Core || pkgreq === Base)
31313128
push!(concrete_deps, pkgreq => module_build_id(modreq))
31323129
end

test/loading.jl

-19
Original file line numberDiff line numberDiff line change
@@ -1129,25 +1129,6 @@ end
11291129
run(cmd_proj_ext)
11301130
end
11311131

1132-
# Sysimage extensions
1133-
# The test below requires that LinearAlgebra is in the sysimage and that it has not been loaded yet.
1134-
# if it gets moved out, this test will need to be updated.
1135-
# We run this test in a new process so we are not vulnerable to a previous test having loaded LinearAlgebra
1136-
sysimg_ext_test_code = """
1137-
uuid_key = Base.PkgId(Base.UUID("37e2e46d-f89d-539d-b4ee-838fcccc9c8e"), "LinearAlgebra")
1138-
Base.in_sysimage(uuid_key) || error("LinearAlgebra not in sysimage")
1139-
haskey(Base.explicit_loaded_modules, uuid_key) && error("LinearAlgebra already loaded")
1140-
using HasExtensions
1141-
Base.get_extension(HasExtensions, :LinearAlgebraExt) === nothing || error("unexpectedly got an extension")
1142-
using LinearAlgebra
1143-
haskey(Base.explicit_loaded_modules, uuid_key) || error("LinearAlgebra not loaded")
1144-
Base.get_extension(HasExtensions, :LinearAlgebraExt) isa Module || error("expected extension to load")
1145-
"""
1146-
cmd = `$(Base.julia_cmd()) --startup-file=no -e $sysimg_ext_test_code`
1147-
cmd = addenv(cmd, "JULIA_LOAD_PATH" => join([proj, "@stdlib"], sep))
1148-
run(cmd)
1149-
1150-
11511132
# Extensions in implicit environments
11521133
old_load_path = copy(LOAD_PATH)
11531134
try

test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml

+1-6
Original file line numberDiff line numberDiff line change
@@ -25,17 +25,12 @@ deps = ["ExtDep3"]
2525
path = "../HasExtensions.jl"
2626
uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8"
2727
version = "0.1.0"
28+
weakdeps = ["ExtDep", "ExtDep2"]
2829

2930
[deps.HasExtensions.extensions]
3031
Extension = "ExtDep"
3132
ExtensionDep = "ExtDep3"
3233
ExtensionFolder = ["ExtDep", "ExtDep2"]
33-
LinearAlgebraExt = "LinearAlgebra"
34-
35-
[deps.HasExtensions.weakdeps]
36-
ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c"
37-
ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d"
38-
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
3934

4035
[[deps.SomePackage]]
4136
path = "../SomePackage"

test/project/Extensions/HasExtensions.jl/Project.toml

-2
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,8 @@ ExtDep3 = "a5541f1e-a556-4fdc-af15-097880d743a1"
88
[weakdeps]
99
ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c"
1010
ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d"
11-
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
1211

1312
[extensions]
1413
Extension = "ExtDep"
1514
ExtensionDep = "ExtDep3"
1615
ExtensionFolder = ["ExtDep", "ExtDep2"]
17-
LinearAlgebraExt = "LinearAlgebra"

test/project/Extensions/HasExtensions.jl/ext/LinearAlgebraExt.jl

-3
This file was deleted.

0 commit comments

Comments
 (0)