serialize from workers, then collate at end into .jsons

Author: IanButterworth

test/buildkitetestjson.jl

@@ -7,8 +7,9 @@ module BuildkiteTestJSON
 
 using Test
 using Dates
+using Serialization
 
-export write_testset_json_files
+export serialize_testset_result_file, write_testset_json_files
 
 # Bootleg JSON writer
 
@@ -205,17 +206,37 @@ function collect_results!(results::Vector{Dict{String,Any}}, testset::Test.Defau
     return results
 end
 
-function write_testset_json_files(dir::String, testset::Test.DefaultTestSet)
+function serialize_testset_result_file(dir::String, testset::Test.DefaultTestSet)
     data = Dict{String,Any}[]
     collect_results!(data, testset)
+    name = replace(testset.description, r"[^a-zA-Z0-9]" => "_")
+    res_file = joinpath(dir, "results_$(name).dat")
+    Serialization.serialize(res_file, data)
+    @debug "Saved $(basename(res_file)) ($(Base.format_bytes(filesize(res_file))))"
+    return res_file
+end
+
+# deserilalizes the results files and writes them to collated JSON files of 5000 max results
+function write_testset_json_files(dir::String)
+    data = Dict{String,Any}[]
+    read_files = String[]
+    for res_dat in filter!(x -> occursin(r"^results.*\.dat$", x), readdir(dir))
+        res_file = joinpath(dir, res_dat)
+        append!(data, Serialization.deserialize(res_file))
+        @debug "Loaded $(basename(res_file)) ($(Base.format_bytes(filesize(res_file))))"
+        push!(read_files, res_file)
+    end
     files = String[]
     # Buildkite is limited to 5000 results per file https://buildkite.com/docs/test-analytics/importing-json
     for (i, chunk) in enumerate(Iterators.partition(data, 5000))
-        name = replace(testset.description, r"[^a-zA-Z0-9]" => "_")
-        res_file = joinpath(dir, "results_$(name)_$(i).json")
+        res_file = joinpath(dir, "results_$(i).json")
         open(io -> json_repr(io, chunk), res_file, "w")
         push!(files, res_file)
-        @debug "Saved $(basename(res_file)) ($(Base.format_bytes(filesize(res_file))))"
+        @info "Saved $(basename(res_file)) ($(length(chunk)) results, $(Base.format_bytes(filesize(res_file))))"
+    end
+    for res_file in read_files
+        rm(res_file)
+        @debug "Deleted $(basename(res_file))"
     end
     return files
 end

test/runtests.jl

@@ -11,6 +11,8 @@ using Base: Experimental
 
 include("choosetests.jl")
 include("testenv.jl")
+include("buildkitetestjson.jl")
+using .BuildkiteTestJSON
 
 (; tests, net_on, exit_on_error, use_revise, seed) = choosetests(ARGS)
 tests = unique(tests)
@@ -341,6 +343,8 @@ cd(@__DIR__) do
         end
     end
 
+    write_testset_json_files(@__DIR__)
+
     #=
 `   Construct a testset on the master node which will hold results from all the
     test files run on workers and on node1. The loop goes through the results,

test/testdefs.jl

@@ -91,11 +91,9 @@ function runtests(name, path, isolate=true; seed=nothing)
         if Base.get_bool_env("CI", false)
             # @info "Writing test result data to $(@__DIR__)"
             # Profile.clear()
-            t = @elapsed result_files = write_testset_json_files(@__DIR__, ts)
+            t = @elapsed result_file = serialize_testset_result_file(@__DIR__, ts)
             if t > 5
-                sizes = map(f -> Base.format_bytes(filesize(f)), result_files)
-                result_files_info = Dict(zip(result_files, sizes))
-                @warn "Writing test result data was slow: $t seconds" result_files_info
+                @warn "Writing test result data was slow: $t seconds" file=result_file size=Base.format_bytes(filesize(result_file))
             end
             # Profile.print(IOContext(stdout, :displaysize=>(1000,200)), noisefloor=1.0)
         end