Save results from workers, then collate into jsons at end of Base.runtests

Author: IanButterworth

test/runtests.jl

@@ -13,8 +13,6 @@ include("choosetests.jl")
 include("testenv.jl")
 include("buildkitetestjson.jl")
 
-using .BuildkiteTestJSON
-
 (; tests, net_on, exit_on_error, use_revise, seed) = choosetests(ARGS)
 tests = unique(tests)
 
@@ -344,6 +342,8 @@ cd(@__DIR__) do
         end
     end
 
+    BuildkiteTestJSON.write_testset_json_files(@__DIR__)
+
     #=
 `   Construct a testset on the master node which will hold results from all the
     test files run on workers and on node1. The loop goes through the results,
@@ -418,11 +418,6 @@ cd(@__DIR__) do
         Test.pop_testset()
     end
 
-    if Base.get_bool_env("CI", false)
-        @info "Writing test result data to $(@__DIR__)"
-        write_testset_json_files(@__DIR__, o_ts)
-    end
-
     Test.TESTSET_PRINT_ENABLE[] = true
     println()
     # o_ts.verbose = true # set to true to show all timings when successful

test/testdefs.jl

@@ -2,12 +2,15 @@
 
 using Test, Random
 
+include("buildkitetestjson.jl")
+
 function runtests(name, path, isolate=true; seed=nothing)
     old_print_setting = Test.TESTSET_PRINT_ENABLE[]
     Test.TESTSET_PRINT_ENABLE[] = false
     # remove all hint_handlers, so that errorshow tests are not changed by which packages have been loaded on this worker already
     # packages that call register_error_hint should also call this again, and then re-add any hooks they want to test
     empty!(Base.Experimental._hint_handlers)
+    withenv("JULIA_TEST_RECORD_PASSES" => Base.get_bool_env("CI", false)) do
     try
         if isolate
             # Simple enough to type and random enough so that no one will hard
@@ -30,7 +33,13 @@ function runtests(name, path, isolate=true; seed=nothing)
             original_env = copy(ENV)
             original_project = Base.active_project()
 
-            Base.include(m, "$path.jl")
+            try
+                Base.include(m, "$path.jl")
+            finally
+                if Base.get_bool_env("CI", false)
+                    BuildkiteTestJSON.serialize_testset_result_file(@__DIR__, Test.get_testset())
+                end
+            end
 
             if Base.DEPOT_PATH != original_depot_path
                 msg = "The `$(name)` test set mutated Base.DEPOT_PATH and did not restore the original values"
@@ -98,6 +107,7 @@ function runtests(name, path, isolate=true; seed=nothing)
         ex isa TestSetException || rethrow()
         return Any[ex]
     end
+    end # withenv
 end
 
 # looking in . messes things up badly