From 7bb938e9fc56f8252af253650f731fb263986bdd Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 14 Apr 2017 11:04:42 -0500 Subject: [PATCH 1/4] Remove need for temp_rel_pkg_dir --- test/pkg.jl | 34 +++++++++------------------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/test/pkg.jl b/test/pkg.jl index ee9fc64103b33..bbe361d345934 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -18,10 +18,11 @@ function capture_stdout(f::Function) end -function temp_pkg_dir(fn::Function, remove_tmp_dir::Bool=true) +function temp_pkg_dir(fn::Function, tmp_dir=joinpath(tempdir(), randstring()), + remove_tmp_dir::Bool=true) + # Used in tests below to set up and tear down a sandboxed package directory - const tmpdir = joinpath(tempdir(),randstring()) - withenv("JULIA_PKGDIR" => tmpdir) do + withenv("JULIA_PKGDIR" => tmp_dir) do @test !isdir(Pkg.dir()) try Pkg.init() @@ -29,7 +30,7 @@ function temp_pkg_dir(fn::Function, remove_tmp_dir::Bool=true) Pkg.resolve() fn() finally - remove_tmp_dir && rm(tmpdir, recursive=true) + remove_tmp_dir && rm(tmp_dir, recursive=true) end end end @@ -569,28 +570,11 @@ let io = IOBuffer() @test !contains(String(take!(io)), "backtrace()") end - -function temp_rel_pkg_dir(fn::Function, remove_tmp_dir::Bool=true) - # Used in tests below to set up and tear down a sandboxed package directory +@testset "Relative path operations" begin cd(tempdir()) do - const tmpdir = randstring() - withenv("JULIA_PKGDIR" => tmpdir) do - @test !isdir(Pkg.dir()) - try - Pkg.init() - @test isdir(Pkg.dir()) - Pkg.resolve() - fn() - finally - remove_tmp_dir && rm(tmpdir, recursive=true) - end + temp_pkg_dir(randstring()) do + Pkg.add("Example") + @test [keys(Pkg.installed())...] == ["Example"] end end end - -@testset "Relative path operations" begin - temp_rel_pkg_dir() do - Pkg.add("Example") - @test [keys(Pkg.installed())...] == ["Example"] - end -end From 94f1f8530402ab830bc31ffcdbc4192fd905fb35 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Sat, 14 Jan 2017 12:03:51 -0600 Subject: [PATCH 2/4] Exit calls in build scripts no longer abort build --- base/pkg/entry.jl | 26 ++++++++++++++++---------- test/pkg.jl | 42 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 13f9abcb6d867..63ff84ae05475 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -609,19 +609,25 @@ function build!(pkgs::Vector, errs::Dict, seen::Set=Set()) empty!(Base.DL_LOAD_PATH) append!(Base.DL_LOAD_PATH, $(repr(Base.DL_LOAD_PATH))) open("$(escape_string(errfile))", "a") do f - for path in eachline(STDIN) - pkg = basename(dirname(dirname(path))) - try - info("Building \$pkg") - cd(dirname(path)) do - evalfile(path) + pkg = "" + atexit(() -> !isempty(pkg) && run_build()) + function run_build() + for path in eachline(STDIN) + pkg = basename(dirname(dirname(path))) + try + info("Building \$pkg") + cd(dirname(path)) do + evalfile(path) + end + catch err + Base.Pkg.Entry.warnbanner(err, label="[ ERROR: \$pkg ]") + serialize(f, pkg) + serialize(f, err) end - catch err - Base.Pkg.Entry.warnbanner(err, label="[ ERROR: \$pkg ]") - serialize(f, pkg) - serialize(f, err) end end + run_build() + pkg = "" end """ io, pobj = open(pipeline(detach(`$(Base.julia_cmd()) -O0 diff --git a/test/pkg.jl b/test/pkg.jl index bbe361d345934..afbf89adef932 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -19,15 +19,19 @@ end function temp_pkg_dir(fn::Function, tmp_dir=joinpath(tempdir(), randstring()), - remove_tmp_dir::Bool=true) + remove_tmp_dir::Bool=true; initialize::Bool=true) # Used in tests below to set up and tear down a sandboxed package directory withenv("JULIA_PKGDIR" => tmp_dir) do @test !isdir(Pkg.dir()) try - Pkg.init() - @test isdir(Pkg.dir()) - Pkg.resolve() + if initialize + Pkg.init() + @test isdir(Pkg.dir()) + Pkg.resolve() + else + mkpath(Pkg.dir()) + end fn() finally remove_tmp_dir && rm(tmp_dir, recursive=true) @@ -578,3 +582,33 @@ end end end end + +temp_pkg_dir(initialize=false) do + function write_build(pkg, content) + build_filename = Pkg.dir(pkg, "deps", "build.jl") + mkpath(dirname(build_filename)) + write(build_filename, content) + end + + write_build("Normal", "") + write_build("Error", "error(\"An error has occurred while building a package\")") + write_build("Exit", "exit()") + + cd(Pkg.dir()) do + errors = Dict() + + empty!(errors) + @test_warn ("INFO: Building Error", + "INFO: Building Normal") Pkg.Entry.build!(["Error", "Normal"], errors) + + empty!(errors) + @test_warn ("INFO: Building Exit", + "INFO: Building Normal") Pkg.Entry.build!(["Exit", "Normal"], errors) + + empty!(errors) + @test_warn ("INFO: Building Exit", + "INFO: Building Normal", + "INFO: Building Exit", + "INFO: Building Normal") Pkg.Entry.build!(["Exit", "Normal", "Exit", "Normal"], errors) + end +end From 7a3c2fe383b257a2b3ceeeff9215941269f6126d Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 24 Jan 2017 11:09:40 -0600 Subject: [PATCH 3/4] Isolate each build.jl in a seperate Julia process --- base/pkg/entry.jl | 87 ++++++++++++++++++++--------------------------- 1 file changed, 37 insertions(+), 50 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 63ff84ae05475..d5fea32de717c 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -579,26 +579,9 @@ function warnbanner(msg...; label="[ WARNING ]", prefix="") warn(prefix="", "="^cols) end -function build!(pkgs::Vector, buildstream::IO, seen::Set) - for pkg in pkgs - pkg == "julia" && continue - pkg in seen ? continue : push!(seen,pkg) - Read.isinstalled(pkg) || throw(PkgError("$pkg is not an installed package")) - build!(Read.requires_list(pkg),buildstream,seen) - path = abspath(pkg,"deps","build.jl") - isfile(path) || continue - println(buildstream, path) # send to build process for evalfile - flush(buildstream) - end -end - -function build!(pkgs::Vector, errs::Dict, seen::Set=Set()) - # To isolate the build from the running Julia process, we - # execute the build.jl files in a separate process that - # is sitting there waiting for paths to evaluate. Errors - # are serialized to errfile for later retrieval into errs[pkg] - errfile = tempname() - close(open(errfile, "w")) # create empty file +function build(pkg::AbstractString, build_file::AbstractString, errfile::AbstractString) + # To isolate the build from the running Julia process, we execute each build.jl file in + # a separate process. Errors are serialized to errfile for later reporting. # TODO: serialize the same way the load cache does, not with strings LOAD_PATH = filter(x -> x isa AbstractString, Base.LOAD_PATH) code = """ @@ -609,37 +592,45 @@ function build!(pkgs::Vector, errs::Dict, seen::Set=Set()) empty!(Base.DL_LOAD_PATH) append!(Base.DL_LOAD_PATH, $(repr(Base.DL_LOAD_PATH))) open("$(escape_string(errfile))", "a") do f - pkg = "" - atexit(() -> !isempty(pkg) && run_build()) - function run_build() - for path in eachline(STDIN) - pkg = basename(dirname(dirname(path))) - try - info("Building \$pkg") - cd(dirname(path)) do - evalfile(path) - end - catch err - Base.Pkg.Entry.warnbanner(err, label="[ ERROR: \$pkg ]") - serialize(f, pkg) - serialize(f, err) - end + pkg, build_file = "$pkg", "$(escape_string(build_file))" + try + info("Building \$pkg") + cd(dirname(build_file)) do + evalfile(build_file) end + catch err + Base.Pkg.Entry.warnbanner(err, label="[ ERROR: \$pkg ]") + serialize(f, pkg) + serialize(f, err) end - run_build() - pkg = "" end """ - io, pobj = open(pipeline(detach(`$(Base.julia_cmd()) -O0 - --compilecache=$(Bool(Base.JLOptions().use_compilecache) ? "yes" : "no") - --history-file=no - --color=$(Base.have_color ? "yes" : "no") - --eval $code`), stderr=STDERR), "w", STDOUT) + cmd = `$(Base.julia_cmd()) -O0 + --compilecache=$(Bool(Base.JLOptions().use_compilecache) ? "yes" : "no") + --history-file=no + --color=$(Base.have_color ? "yes" : "no") + --eval $code` + + success(pipeline(cmd, stderr=STDERR)) +end + +function build!(pkgs::Vector, seen::Set, errfile::AbstractString) + for pkg in pkgs + pkg == "julia" && continue + pkg in seen ? continue : push!(seen,pkg) + Read.isinstalled(pkg) || throw(PkgError("$pkg is not an installed package")) + build!(Read.requires_list(pkg), seen, errfile) + path = abspath(pkg,"deps","build.jl") + isfile(path) || continue + build(pkg, path, errfile) || error("Build process failed.") + end +end + +function build!(pkgs::Vector, errs::Dict, seen::Set=Set()) + errfile = tempname() + touch(errfile) # create empty file try - build!(pkgs, io, seen) - close(io) - wait(pobj) - success(pobj) || error("Build process failed.") + build!(pkgs, seen, errfile) open(errfile, "r") do f while !eof(f) pkg = deserialize(f) @@ -647,10 +638,6 @@ function build!(pkgs::Vector, errs::Dict, seen::Set=Set()) errs[pkg] = err end end - catch err - close(io) - isa(err, PkgError) ? wait(pobj) : kill(pobj) - rethrow(err) finally isfile(errfile) && Base.rm(errfile) end From c3063d4eb6f0849ebf15c326d57c3fb8a39c95d3 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 14 Apr 2017 11:21:43 -0500 Subject: [PATCH 4/4] Switch to triple-backticks --- base/pkg/entry.jl | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index d5fea32de717c..b794d25012f3b 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -605,11 +605,13 @@ function build(pkg::AbstractString, build_file::AbstractString, errfile::Abstrac end end """ - cmd = `$(Base.julia_cmd()) -O0 - --compilecache=$(Bool(Base.JLOptions().use_compilecache) ? "yes" : "no") - --history-file=no - --color=$(Base.have_color ? "yes" : "no") - --eval $code` + cmd = ``` + $(Base.julia_cmd()) -O0 + --compilecache=$(Bool(Base.JLOptions().use_compilecache) ? "yes" : "no") + --history-file=no + --color=$(Base.have_color ? "yes" : "no") + --eval $code + ``` success(pipeline(cmd, stderr=STDERR)) end