Don't bail out on test errors by default

Add JL_RUNTESTS_STOP environment variable to control fail-on-error
behavior for the CI systems.

Author: ihnorton

.travis.yml

@@ -62,6 +62,6 @@ script:
     - cd .. && mv julia julia2
     - cp /tmp/julia/lib/julia/sys.ji local.ji && /tmp/julia/bin/julia-debug -J local.ji -e 'true' && rm local.ji
     - /tmp/julia/bin/julia-debug -e 'versioninfo()'
-    - cd /tmp/julia/share/julia/test && /tmp/julia/bin/julia-debug --check-bounds=yes runtests.jl all && /tmp/julia/bin/julia-debug --check-bounds=yes runtests.jl pkg
+    - cd /tmp/julia/share/julia/test && /tmp/julia/bin/julia-debug --check-bounds=yes runtests.jl all && JL_TESTFAILURE_STOP=1 /tmp/julia/bin/julia-debug --check-bounds=yes runtests.jl pkg
     - cd - && mv julia2 julia
     - echo "Ready for packaging..."

appveyor.yml

@@ -50,4 +50,4 @@ build_script:
 
 test_script:
   - usr\bin\julia -e "versioninfo()"
-  - cd test && ..\usr\bin\julia runtests.jl all
+  - cd test && JL_TESTFAILURE_STOP=1 ..\usr\bin\julia runtests.jl all

test/runtests.jl

@@ -10,8 +10,10 @@ cd(dirname(@__FILE__)) do
 
     @everywhere include("testdefs.jl")
 
-    reduce(propagate_errors, nothing, pmap(runtests, tests; err_retry=false, err_stop=true))
+    err_stop = haskey(ENV, "JL_TESTFAILURE_STOP")
+
+    results = reduce(propagate_errors, nothing, pmap(runtests, tests; err_retry=false, err_stop=err_stop))
 
     @unix_only n > 1 && rmprocs(workers(), waitfor=5.0)
-    println("    \033[32;1mSUCCESS\033[0m")
+    all(x->x==false, results) && println("    \033[32;1mSUCCESS\033[0m")
 end

test/testdefs.jl

@@ -8,13 +8,18 @@ function runtests(name)
 end
 
 function propagate_errors(a,b)
-    if isa(a,Exception)
+    err_stop = haskey(ENV, "JL_TESTFAILURE_STOP")
+    err = false
+    if isa(a, Exception) || isa(b, Exception)
+        err = true
+    end
+    if isa(a,Exception) && err_stop
         rethrow(a)
     end
-    if isa(b,Exception)
+    if isa(b,Exception) && err_stop
         rethrow(b)
     end
-    nothing
+    err
 end
 
 # looking in . messes things up badly