From 63fb25ce5282dd0bb9b963d41714fcb6539a1b2e Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 18 Mar 2025 14:07:08 -0500 Subject: [PATCH 01/12] Show evaluated test arguments for functions with 1 & 2 arguments --- stdlib/Test/src/Test.jl | 6 +++++- stdlib/Test/test/runtests.jl | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index e8d670b3d7d00..9d0043bc78959 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -30,6 +30,7 @@ using InteractiveUtils: gen_call_with_extracted_types using Base: typesplit, remove_linenums! using Serialization: Serialization +# Whitelist boolean functions which show their evaluated arguments when the test fails const DISPLAY_FAILED = ( :isequal, :isapprox, @@ -619,7 +620,8 @@ function get_test_result(ex, source) $(QuoteNode(source)), $negate, )) - elseif isa(ex, Expr) && ex.head === :call && ex.args[1] in DISPLAY_FAILED + elseif isa(ex, Expr) && ex.head === :call && + (2 <= length(ex.args) <= 3 || ex.args[1] in DISPLAY_FAILED) escaped_func = esc(ex.args[1]) quoted_func = QuoteNode(ex.args[1]) @@ -1623,6 +1625,7 @@ julia> @testset let logi = log(im) end Test Failed at none:3 Expression: !(iszero(real(logi))) + Evaluated: !(iszero(0.0)) Context: logi = 0.0 + 1.5707963267948966im ERROR: There was an error during testing @@ -1633,6 +1636,7 @@ julia> @testset let logi = log(im), op = !iszero end Test Failed at none:3 Expression: op(real(logi)) + Evaluated: op(0.0) Context: logi = 0.0 + 1.5707963267948966im op = !iszero diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index 02523dc6fd911..74a16f9731d0c 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -163,6 +163,10 @@ let fails = @testset NoThrowTestSet begin @test_throws r"sqrt\([Cc]omplx" sqrt(-1) @test_throws str->occursin("a T", str) error("a test") @test_throws ["BoundsError", "acquire", "1-element", "at index [2]"] [1][2] + # 27 - Fail - issetequal + a = [1, 2] + b = [1, 3] + @test issetequal(a, b) end for fail in fails @test fail isa Test.Fail @@ -298,6 +302,12 @@ let fails = @testset NoThrowTestSet begin @test occursin(r"Message: \"BoundsError.* 1-element.*at index \[2\]", str) end + let str = sprint(show, fails[27]) + @test !(:issetequal in Test.DISPLAY_FAILED) + @test occursin("Expression: issetequal(a, b)", str) + @test occursin("Evaluated: issetequal([1, 2], [1, 3])", str) + end + end struct BadError <: Exception end From c46f28c5776f108002ce63f00c0fc97143b2b839 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 19 Mar 2025 09:15:27 -0500 Subject: [PATCH 02/12] Avoid printing test records during testing Printing resulted from calling `record` on a `Test.Error` and `Test.Fail` was not actually tested at this point and was just creating noise in our test results. I've confirmed that later tests to at least execute the `print_result` code path. --- stdlib/Test/test/runtests.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index 74a16f9731d0c..df64d0c5da0c3 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -355,9 +355,9 @@ let retval_tests = @testset NoThrowTestSet begin pass_mock = Test.Pass(:test, 1, 2, 3, LineNumberNode(0, "A Pass Mock")) @test Test.record(ts, pass_mock) isa Test.Pass error_mock = Test.Error(:test, 1, 2, 3, LineNumberNode(0, "An Error Mock")) - @test Test.record(ts, error_mock) isa Test.Error + @test Test.record(ts, error_mock; print_result=false) isa Test.Error fail_mock = Test.Fail(:test, 1, 2, 3, nothing, LineNumberNode(0, "A Fail Mock"), false) - @test Test.record(ts, fail_mock) isa Test.Fail + @test Test.record(ts, fail_mock; print_result=false) isa Test.Fail broken_mock = Test.Broken(:test, LineNumberNode(0, "A Broken Mock")) @test Test.record(ts, broken_mock) isa Test.Broken end From f8beb88e4509cc064548913f66dbb2f24714812a Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 19 Mar 2025 09:18:22 -0500 Subject: [PATCH 03/12] Use the term deprecated in depwarn Use of `@test_deprecated` requires that the deprecation warning uses the term "deprecated" in the warning when using `--depwarn=yes`. --- stdlib/Test/src/Test.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 9d0043bc78959..882b4dd8a3f93 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -818,7 +818,7 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype) Base.depwarn("macroexpand no longer throws a LoadError so `@test_throws LoadError ...` is deprecated and passed without checking the error type!", :do_test_throws) true elseif extype == ErrorException && isa(exc, FieldError) - Base.depwarn(lazy"ErrorException should no longer be used to test field access; FieldError should be used instead!", :do_test_throws) + Base.depwarn(lazy"Using ErrorException to test field access is deprecated; use FieldError instead.", :do_test_throws) true else isa(exc, extype) From cdff5458172332d51d73ce2a4761fbbf43a3eb8c Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 19 Mar 2025 09:42:41 -0500 Subject: [PATCH 04/12] fixup! Show evaluated test arguments for functions with 1 & 2 arguments --- stdlib/Test/test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index df64d0c5da0c3..7dad2e9f4ddaa 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -204,7 +204,7 @@ let fails = @testset NoThrowTestSet begin let str = sprint(show, fails[7]) @test occursin("Expression: (==)(1:2...)", str) - @test !occursin("Evaluated", str) + @test occursin("Evaluated: 1 == 2", str) end let str = sprint(show, fails[8]) From bac403907da1935636f1519988da70b6cac060c1 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 19 Mar 2025 11:33:31 -0500 Subject: [PATCH 05/12] Add `1 == 2` test --- stdlib/Test/test/runtests.jl | 112 ++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 53 deletions(-) diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index 7dad2e9f4ddaa..c07e40aa1538e 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -115,58 +115,59 @@ let fails = @testset NoThrowTestSet begin @test_throws OverflowError error() # 2 - Fail - no exception @test_throws OverflowError 1 + 1 - # 3 - Fail - comparison + # 3 & 4 - Fail - comparison + @test 1 == 2 @test 1+1 == 2+2 - # 4 - Fail - approximate comparison + # 5 - Fail - approximate comparison @test 1/1 ≈ 2/1 - # 5 - Fail - chained comparison + # 6 - Fail - chained comparison @test 1+0 == 2+0 == 3+0 - # 6 - Fail - comparison call + # 7 - Fail - comparison call @test ==(1 - 2, 2 - 1) - # 7 - Fail - splatting + # 8 - Fail - splatting @test ==(1:2...) - # 8 - Fail - isequal + # 9 - Fail - isequal @test isequal(0 / 0, 1 / 0) - # 9 - Fail - function splatting + # 10 - Fail - function splatting @test isequal(1:2...) - # 10 - Fail - isapprox + # 11 - Fail - isapprox @test isapprox(0 / 1, -1 / 0) - # 11 & 12 - Fail - function with keyword + # 12 & 13 - Fail - function with keyword @test isapprox(1 / 2, 2 / 1, atol=1 / 1) @test isapprox(1 - 2, 2 - 1; atol=1 - 1) - # 13 - Fail - function keyword splatting + # 14 - Fail - function keyword splatting k = [(:atol, 0), (:nans, true)] @test isapprox(1, 2; k...) - # 14 - Fail - call negation + # 15 - Fail - call negation @test !isequal(1, 2 - 1) - # 15 - Fail - comparison negation + # 16 - Fail - comparison negation @test !(2 + 3 == 1 + 4) - # 16 - Fail - chained negation + # 17 - Fail - chained negation @test !(2 + 3 == 1 + 4 == 5) - # 17 - Fail - isempty + # 18 - Fail - isempty nonempty = [1, 2, 3] @test isempty(nonempty) str1 = "Hello" str2 = "World" - # 18 - Fail - occursin + # 19 - Fail - occursin @test occursin(str1, str2) - # 19 - Fail - startswith + # 20 - Fail - startswith @test startswith(str1, str2) - # 20 - Fail - endswith + # 21 - Fail - endswith @test endswith(str1, str2) - # 21 - Fail - contains + # 22 - Fail - contains @test contains(str1, str2) - # 22 - Fail - Type Comparison + # 23 - Fail - issetequal + a = [1, 2] + b = [1, 3] + @test issetequal(a, b) + # 24 - Fail - Type Comparison @test typeof(1) <: typeof("julia") - # 23 - 26 - Fail - wrong message + # 27 - 28 - Fail - wrong message @test_throws "A test" error("a test") @test_throws r"sqrt\([Cc]omplx" sqrt(-1) @test_throws str->occursin("a T", str) error("a test") @test_throws ["BoundsError", "acquire", "1-element", "at index [2]"] [1][2] - # 27 - Fail - issetequal - a = [1, 2] - b = [1, 3] - @test issetequal(a, b) end for fail in fails @test fail isa Test.Fail @@ -183,131 +184,136 @@ let fails = @testset NoThrowTestSet begin end let str = sprint(show, fails[3]) + @test occursin("Expression: 1 == 2", str) + @test occursin("Evaluated: 1 == 2", str) + end + + let str = sprint(show, fails[4]) @test occursin("Expression: 1 + 1 == 2 + 2", str) @test occursin("Evaluated: 2 == 4", str) end - let str = sprint(show, fails[4]) + let str = sprint(show, fails[5]) @test occursin("Expression: 1 / 1 ≈ 2 / 1", str) @test occursin("Evaluated: 1.0 ≈ 2.0", str) end - let str = sprint(show, fails[5]) + let str = sprint(show, fails[6]) @test occursin("Expression: 1 + 0 == 2 + 0 == 3 + 0", str) @test occursin("Evaluated: 1 == 2 == 3", str) end - let str = sprint(show, fails[6]) + let str = sprint(show, fails[7]) @test occursin("Expression: 1 - 2 == 2 - 1", str) @test occursin("Evaluated: -1 == 1", str) end - let str = sprint(show, fails[7]) + let str = sprint(show, fails[8]) @test occursin("Expression: (==)(1:2...)", str) @test occursin("Evaluated: 1 == 2", str) end - let str = sprint(show, fails[8]) + let str = sprint(show, fails[9]) @test occursin("Expression: isequal(0 / 0, 1 / 0)", str) @test occursin("Evaluated: isequal(NaN, Inf)", str) end - let str = sprint(show, fails[9]) + let str = sprint(show, fails[10]) @test occursin("Expression: isequal(1:2...)", str) @test occursin("Evaluated: isequal(1, 2)", str) end - let str = sprint(show, fails[10]) + let str = sprint(show, fails[11]) @test occursin("Expression: isapprox(0 / 1, -1 / 0)", str) @test occursin("Evaluated: isapprox(0.0, -Inf)", str) end - let str = sprint(show, fails[11]) + let str = sprint(show, fails[12]) @test occursin("Expression: isapprox(1 / 2, 2 / 1, atol = 1 / 1)", str) @test occursin("Evaluated: isapprox(0.5, 2.0; atol = 1.0)", str) end - let str = sprint(show, fails[12]) + let str = sprint(show, fails[13]) @test occursin("Expression: isapprox(1 - 2, 2 - 1; atol = 1 - 1)", str) @test occursin("Evaluated: isapprox(-1, 1; atol = 0)", str) end - let str = sprint(show, fails[13]) + let str = sprint(show, fails[14]) @test occursin("Expression: isapprox(1, 2; k...)", str) @test occursin("Evaluated: isapprox(1, 2; atol = 0, nans = true)", str) end - let str = sprint(show, fails[14]) + let str = sprint(show, fails[15]) @test occursin("Expression: !(isequal(1, 2 - 1))", str) @test occursin("Evaluated: !(isequal(1, 1))", str) end - let str = sprint(show, fails[15]) + let str = sprint(show, fails[16]) @test occursin("Expression: !(2 + 3 == 1 + 4)", str) @test occursin("Evaluated: !(5 == 5)", str) end - let str = sprint(show, fails[16]) + let str = sprint(show, fails[17]) @test occursin("Expression: !(2 + 3 == 1 + 4 == 5)", str) @test occursin("Evaluated: !(5 == 5 == 5)", str) end - let str = sprint(show, fails[17]) + let str = sprint(show, fails[18]) @test occursin("Expression: isempty(nonempty)", str) @test occursin("Evaluated: isempty([1, 2, 3])", str) end - let str = sprint(show, fails[18]) + let str = sprint(show, fails[19]) @test occursin("Expression: occursin(str1, str2)", str) @test occursin("Evaluated: occursin(\"Hello\", \"World\")", str) end - let str = sprint(show, fails[19]) + let str = sprint(show, fails[20]) @test occursin("Expression: startswith(str1, str2)", str) @test occursin("Evaluated: startswith(\"Hello\", \"World\")", str) end - let str = sprint(show, fails[20]) + let str = sprint(show, fails[21]) @test occursin("Expression: endswith(str1, str2)", str) @test occursin("Evaluated: endswith(\"Hello\", \"World\")", str) end - let str = sprint(show, fails[21]) + let str = sprint(show, fails[22]) @test occursin("Expression: contains(str1, str2)", str) @test occursin("Evaluated: contains(\"Hello\", \"World\")", str) end - let str = sprint(show, fails[22]) + let str = sprint(show, fails[23]) + @test !(:issetequal in Test.DISPLAY_FAILED) + @test occursin("Expression: issetequal(a, b)", str) + @test occursin("Evaluated: issetequal([1, 2], [1, 3])", str) + end + + let str = sprint(show, fails[24]) @test occursin("Expression: typeof(1) <: typeof(\"julia\")", str) @test occursin("Evaluated: $(typeof(1)) <: $(typeof("julia"))", str) end - let str = sprint(show, fails[23]) + let str = sprint(show, fails[25]) @test occursin("Expected: \"A test\"", str) @test occursin("Message: \"a test\"", str) end - let str = sprint(show, fails[24]) + let str = sprint(show, fails[26]) @test occursin("Expected: r\"sqrt\\([Cc]omplx\"", str) @test occursin(r"Message: .*Try sqrt\(Complex", str) end - let str = sprint(show, fails[25]) + let str = sprint(show, fails[27]) @test occursin("Expected: < match function >", str) @test occursin("Message: \"a test\"", str) end - let str = sprint(show, fails[26]) + let str = sprint(show, fails[28]) @test occursin("Expected: [\"BoundsError\", \"acquire\", \"1-element\", \"at index [2]\"]", str) @test occursin(r"Message: \"BoundsError.* 1-element.*at index \[2\]", str) end - let str = sprint(show, fails[27]) - @test !(:issetequal in Test.DISPLAY_FAILED) - @test occursin("Expression: issetequal(a, b)", str) - @test occursin("Evaluated: issetequal([1, 2], [1, 3])", str) - end - end struct BadError <: Exception end From cb970d2e7077f6408302e670703308104a82fae6 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 19 Mar 2025 11:42:54 -0500 Subject: [PATCH 06/12] Skip showing evaluated when expression is identical --- stdlib/Test/src/Test.jl | 2 +- stdlib/Test/test/runtests.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 882b4dd8a3f93..612390a6439d0 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -198,7 +198,7 @@ function Base.show(io::IO, t::Fail) print(io, "\n Expected: ", data) print(io, "\n No exception thrown") elseif t.test_type === :test - if data !== nothing + if data !== nothing && t.orig_expr != data # The test was an expression, so display the term-by-term # evaluated version as well print(io, "\n Evaluated: ", data) diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index c07e40aa1538e..59f53274eeef5 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -185,7 +185,7 @@ let fails = @testset NoThrowTestSet begin let str = sprint(show, fails[3]) @test occursin("Expression: 1 == 2", str) - @test occursin("Evaluated: 1 == 2", str) + @test !occursin("Evaluated", str) end let str = sprint(show, fails[4]) From 1df6837c44969306310522f8dc190757d467f7df Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 19 Mar 2025 11:53:12 -0500 Subject: [PATCH 07/12] Show evaluated arguments for functions without keywords --- stdlib/Test/src/Test.jl | 20 +++++++++++++++++++- stdlib/Test/test/runtests.jl | 23 +++++++++++++++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 612390a6439d0..34ccf66735e5a 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -584,6 +584,24 @@ macro test_skip(ex, kws...) return :(record(get_testset(), $testres)) end +function _is_simple_call(@nospecialize ex) + ex.head === :call || return false + num_pargs = 0 + for x in ex.args[2:end] + if isa(x, Expr) && x.head === :parameters + return false + elseif isa(x, Expr) && x.head === :kw + return false + elseif isa(x, Expr) && x.head === :... + return false # Unable to deterimine number of arguments + else + num_pargs += 1 + end + end + + return 1 <= num_pargs <= 2 +end + # An internal function, called by the code generated by the @test # macro to get results of the test expression. # In the special case of a comparison, e.g. x == 5, generate code to @@ -621,7 +639,7 @@ function get_test_result(ex, source) $negate, )) elseif isa(ex, Expr) && ex.head === :call && - (2 <= length(ex.args) <= 3 || ex.args[1] in DISPLAY_FAILED) + (_is_simple_call(ex) || ex.args[1] in DISPLAY_FAILED) escaped_func = esc(ex.args[1]) quoted_func = QuoteNode(ex.args[1]) diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index 59f53274eeef5..237d0d0ea7f6c 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license using Test, Random -using Test: guardseed +using Test: guardseed, _is_simple_call using Serialization using Distributed: RemoteException @@ -210,7 +210,7 @@ let fails = @testset NoThrowTestSet begin let str = sprint(show, fails[8]) @test occursin("Expression: (==)(1:2...)", str) - @test occursin("Evaluated: 1 == 2", str) + @test !occursin("Evaluated", str) end let str = sprint(show, fails[9]) @@ -1786,3 +1786,22 @@ end end end end + +@testset "_is_simple_function" begin + @test !_is_simple_call(:(f())) + @test _is_simple_call(:(f(x))) + @test _is_simple_call(:(f(x, y))) + @test !_is_simple_call(:(f(x, y, z))) + + @test !_is_simple_call(:(f(; x))) + @test !_is_simple_call(:(f(; x, y))) + @test !_is_simple_call(:(f(x=1))) + @test !_is_simple_call(:(f(x=1, y=2))) + + @test !_is_simple_call(:(f(x, y=1))) + @test !_is_simple_call(:(f(x; y=1))) + + @test !_is_simple_call(:(f(x...))) + @test !_is_simple_call(:(f(x, y...))) + @test !_is_simple_call(:(f(x; y...))) +end From a3ee33cd0ba247d0bc9041a885e27e45ad3ece23 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 19 Mar 2025 12:04:03 -0500 Subject: [PATCH 08/12] Whitelisted functions can be qualified with module --- stdlib/Test/src/Test.jl | 11 ++++++++++- stdlib/Test/test/runtests.jl | 14 ++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 34ccf66735e5a..dc604c8f2dd95 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -602,6 +602,15 @@ function _is_simple_call(@nospecialize ex) return 1 <= num_pargs <= 2 end +function _function_name(@nospecialize ex) + ex.head === :call || throw(ArgumentError("$ex is not a call expression")) + if isa(ex.args[1], Expr) && ex.args[1].head === :. + ex.args[1].args[2].value + else + ex.args[1] + end +end + # An internal function, called by the code generated by the @test # macro to get results of the test expression. # In the special case of a comparison, e.g. x == 5, generate code to @@ -639,7 +648,7 @@ function get_test_result(ex, source) $negate, )) elseif isa(ex, Expr) && ex.head === :call && - (_is_simple_call(ex) || ex.args[1] in DISPLAY_FAILED) + (_is_simple_call(ex) || _function_name(ex) in DISPLAY_FAILED) escaped_func = esc(ex.args[1]) quoted_func = QuoteNode(ex.args[1]) diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index 237d0d0ea7f6c..a3f54b58392d8 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license using Test, Random -using Test: guardseed, _is_simple_call +using Test: guardseed, _is_simple_call, _function_name using Serialization using Distributed: RemoteException @@ -156,7 +156,7 @@ let fails = @testset NoThrowTestSet begin # 21 - Fail - endswith @test endswith(str1, str2) # 22 - Fail - contains - @test contains(str1, str2) + @test Base.contains(str1, str2) # 23 - Fail - issetequal a = [1, 2] b = [1, 3] @@ -279,8 +279,8 @@ let fails = @testset NoThrowTestSet begin end let str = sprint(show, fails[22]) - @test occursin("Expression: contains(str1, str2)", str) - @test occursin("Evaluated: contains(\"Hello\", \"World\")", str) + @test occursin("Expression: Base.contains(str1, str2)", str) + @test occursin("Evaluated: Base.contains(\"Hello\", \"World\")", str) end let str = sprint(show, fails[23]) @@ -1805,3 +1805,9 @@ end @test !_is_simple_call(:(f(x, y...))) @test !_is_simple_call(:(f(x; y...))) end + +@testset "_function_name" begin + @test _function_name(:(isequal(x, y))) === :isequal + @test _function_name(:(Base.isequal(x.y))) === :isequal + @test _function_name(:(Base.Meta.isexpr(ex))) === :isexpr +end From 6277473dbcbfddc1d66ff61a43b01f7fa3301d71 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 19 Mar 2025 12:18:40 -0500 Subject: [PATCH 09/12] fixup! Show evaluated arguments for functions without keywords --- stdlib/Test/src/Test.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index dc604c8f2dd95..238dc6cda7c1d 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -593,7 +593,7 @@ function _is_simple_call(@nospecialize ex) elseif isa(x, Expr) && x.head === :kw return false elseif isa(x, Expr) && x.head === :... - return false # Unable to deterimine number of arguments + return false # Unable to determine number of arguments else num_pargs += 1 end From 9da94fb0c867fa2aca5999f7f06eac921ecdd012 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 19 Mar 2025 13:32:19 -0500 Subject: [PATCH 10/12] fixup! Skip showing evaluated when expression is identical --- stdlib/Test/docs/src/index.md | 1 - 1 file changed, 1 deletion(-) diff --git a/stdlib/Test/docs/src/index.md b/stdlib/Test/docs/src/index.md index 4989baa31d55e..7451f8434f4fa 100644 --- a/stdlib/Test/docs/src/index.md +++ b/stdlib/Test/docs/src/index.md @@ -230,7 +230,6 @@ Test Passed julia> @test 1 ≈ 0.999999 Test Failed at none:1 Expression: 1 ≈ 0.999999 - Evaluated: 1 ≈ 0.999999 ERROR: There was an error during testing ``` From c2627ad4a868c92ef2f70f3735ad1990c10ffef0 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 19 Mar 2025 15:26:20 -0500 Subject: [PATCH 11/12] Exclude broadcasted functions --- stdlib/Test/src/Test.jl | 4 ++++ stdlib/Test/test/runtests.jl | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 238dc6cda7c1d..abf377b17a508 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -586,6 +586,10 @@ end function _is_simple_call(@nospecialize ex) ex.head === :call || return false + + # Broadcasted functions are not currently supported + first(string(ex.args[1])) != '.' || return false + num_pargs = 0 for x in ex.args[2:end] if isa(x, Expr) && x.head === :parameters diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index a3f54b58392d8..5c2de9a0c3b84 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -168,6 +168,8 @@ let fails = @testset NoThrowTestSet begin @test_throws r"sqrt\([Cc]omplx" sqrt(-1) @test_throws str->occursin("a T", str) error("a test") @test_throws ["BoundsError", "acquire", "1-element", "at index [2]"] [1][2] + # 29 - Fail - broadcast + @test 1 .== 2 end for fail in fails @test fail isa Test.Fail @@ -314,6 +316,11 @@ let fails = @testset NoThrowTestSet begin @test occursin(r"Message: \"BoundsError.* 1-element.*at index \[2\]", str) end + let str = sprint(show, fails[29]) + @test occursin("Expression: 1 .== 2", str) + @test !occursin("Evaluated", str) + end + end struct BadError <: Exception end @@ -1804,6 +1811,9 @@ end @test !_is_simple_call(:(f(x...))) @test !_is_simple_call(:(f(x, y...))) @test !_is_simple_call(:(f(x; y...))) + + @test _is_simple_call(:(x == y)) + @test !_is_simple_call(:(x .== y)) end @testset "_function_name" begin From 4997f7ef61c5b6ce4ca05a2af25cf51730c1f5c4 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 21 Mar 2025 09:16:13 -0500 Subject: [PATCH 12/12] Show evaluated arguments for most function calls --- stdlib/Test/src/Test.jl | 42 ++++-------------------------------- stdlib/Test/test/runtests.jl | 38 +++++++++----------------------- 2 files changed, 14 insertions(+), 66 deletions(-) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index abf377b17a508..a1858ff8a213c 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -30,18 +30,6 @@ using InteractiveUtils: gen_call_with_extracted_types using Base: typesplit, remove_linenums! using Serialization: Serialization -# Whitelist boolean functions which show their evaluated arguments when the test fails -const DISPLAY_FAILED = ( - :isequal, - :isapprox, - :≈, - :occursin, - :startswith, - :endswith, - :isempty, - :contains -) - const FAIL_FAST = Ref{Bool}(false) #----------------------------------------------------------------------- @@ -584,35 +572,14 @@ macro test_skip(ex, kws...) return :(record(get_testset(), $testres)) end -function _is_simple_call(@nospecialize ex) +function _can_escape_call(@nospecialize ex) ex.head === :call || return false # Broadcasted functions are not currently supported first(string(ex.args[1])) != '.' || return false - num_pargs = 0 - for x in ex.args[2:end] - if isa(x, Expr) && x.head === :parameters - return false - elseif isa(x, Expr) && x.head === :kw - return false - elseif isa(x, Expr) && x.head === :... - return false # Unable to determine number of arguments - else - num_pargs += 1 - end - end - - return 1 <= num_pargs <= 2 -end - -function _function_name(@nospecialize ex) - ex.head === :call || throw(ArgumentError("$ex is not a call expression")) - if isa(ex.args[1], Expr) && ex.args[1].head === :. - ex.args[1].args[2].value - else - ex.args[1] - end + # At least one positional argument or keyword + return length(ex.args) > 1 end # An internal function, called by the code generated by the @test @@ -651,8 +618,7 @@ function get_test_result(ex, source) $(QuoteNode(source)), $negate, )) - elseif isa(ex, Expr) && ex.head === :call && - (_is_simple_call(ex) || _function_name(ex) in DISPLAY_FAILED) + elseif isa(ex, Expr) && _can_escape_call(ex) escaped_func = esc(ex.args[1]) quoted_func = QuoteNode(ex.args[1]) diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index 5c2de9a0c3b84..5b2963a0e1ea4 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license using Test, Random -using Test: guardseed, _is_simple_call, _function_name +using Test: guardseed, _can_escape_call using Serialization using Distributed: RemoteException @@ -212,7 +212,7 @@ let fails = @testset NoThrowTestSet begin let str = sprint(show, fails[8]) @test occursin("Expression: (==)(1:2...)", str) - @test !occursin("Evaluated", str) + @test occursin("Evaluated: 1 == 2", str) end let str = sprint(show, fails[9]) @@ -286,7 +286,6 @@ let fails = @testset NoThrowTestSet begin end let str = sprint(show, fails[23]) - @test !(:issetequal in Test.DISPLAY_FAILED) @test occursin("Expression: issetequal(a, b)", str) @test occursin("Evaluated: issetequal([1, 2], [1, 3])", str) end @@ -1794,30 +1793,13 @@ end end end -@testset "_is_simple_function" begin - @test !_is_simple_call(:(f())) - @test _is_simple_call(:(f(x))) - @test _is_simple_call(:(f(x, y))) - @test !_is_simple_call(:(f(x, y, z))) - - @test !_is_simple_call(:(f(; x))) - @test !_is_simple_call(:(f(; x, y))) - @test !_is_simple_call(:(f(x=1))) - @test !_is_simple_call(:(f(x=1, y=2))) - - @test !_is_simple_call(:(f(x, y=1))) - @test !_is_simple_call(:(f(x; y=1))) - - @test !_is_simple_call(:(f(x...))) - @test !_is_simple_call(:(f(x, y...))) - @test !_is_simple_call(:(f(x; y...))) - - @test _is_simple_call(:(x == y)) - @test !_is_simple_call(:(x .== y)) -end +@testset "_can_escape_call" begin + @test !_can_escape_call(:(f())) + @test _can_escape_call(:(f(x))) + @test _can_escape_call(:(f(; x))) + @test _can_escape_call(:(f(x=1))) -@testset "_function_name" begin - @test _function_name(:(isequal(x, y))) === :isequal - @test _function_name(:(Base.isequal(x.y))) === :isequal - @test _function_name(:(Base.Meta.isexpr(ex))) === :isexpr + @test _can_escape_call(:(x == y)) + @test !_can_escape_call(:(x .== y)) + @test !_can_escape_call(:((==).(x, y))) end