From 7570995d37f7ce07f9e155ba029b887fe3bf4878 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 19 Mar 2025 15:57:09 -0500 Subject: [PATCH 1/2] Show evaluated test arguments from broadcast functions --- stdlib/Test/src/Test.jl | 225 +++++++++++++++++++++-------------- stdlib/Test/test/runtests.jl | 166 +++++++++++++++++--------- 2 files changed, 247 insertions(+), 144 deletions(-) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index bfaa3aa82d962..bf3fdb63b3fab 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -340,46 +340,55 @@ struct Threw <: ExecutionResult source::LineNumberNode end -function eval_test(evaluated::Expr, quoted::Expr, source::LineNumberNode, negate::Bool=false) - evaled_args = evaluated.args +function eval_test_comparison(comparisons::Tuple, quoted::Expr, source::LineNumberNode, negate::Bool=false) quoted_args = quoted.args - n = length(evaled_args) + n = length(comparisons) kw_suffix = "" - if evaluated.head === :comparison - args = evaled_args - res = true - i = 1 - while i < n - a, op, b = args[i], args[i+1], args[i+2] - if res - res = op(a, b) - end - quoted_args[i] = a - quoted_args[i+2] = b - i += 2 - end - elseif evaluated.head === :call - op = evaled_args[1] - kwargs = (evaled_args[2]::Expr).args # Keyword arguments from `Expr(:parameters, ...)` - args = evaled_args[3:n] - - res = op(args...; kwargs...) - - # Create "Evaluated" expression which looks like the original call but has all of - # the arguments evaluated - func_sym = quoted_args[1]::Union{Symbol,Expr} - if isempty(kwargs) - quoted = Expr(:call, func_sym, args...) - elseif func_sym === :≈ && !res - quoted = Expr(:call, func_sym, args...) - kw_suffix = " ($(join(["$k=$v" for (k, v) in kwargs], ", ")))" - else - kwargs_expr = Expr(:parameters, [Expr(:kw, k, v) for (k, v) in kwargs]...) - quoted = Expr(:call, func_sym, kwargs_expr, args...) + res = true + i = 1 + while i < n + a, op, b = comparisons[i], comparisons[i+1], comparisons[i+2] + if res + res = op(a, b) end + quoted_args[i] = a + quoted_args[i+2] = b + i += 2 + end + + if negate + res = !res + quoted = Expr(:call, :!, quoted) + end + + Returned(res, + # stringify arguments in case of failure, for easy remote printing + res === true ? quoted : sprint(print, quoted, context=(:limit => true)) * kw_suffix, + source) +end + +function eval_test_function(func, args, kwargs, quoted_func::Union{Expr,Symbol}, source::LineNumberNode, negate::Bool=false) + res = func(args...; kwargs...) + + # Create "Evaluated" expression which looks like the original call but has all of + # the arguments evaluated + kw_suffix = "" + if quoted_func === :≈ && !res + kw_suffix = " ($(join(["$k=$v" for (k, v) in kwargs], ", ")))" + quoted_args = args + elseif isempty(kwargs) + quoted_args = args else - throw(ArgumentError("Unhandled expression type: $(evaluated.head)")) + kwargs_expr = Expr(:parameters, [Expr(:kw, k, v) for (k, v) in kwargs]...) + quoted_args = [kwargs_expr, args...] + end + + # Properly render broadcast function call syntax, e.g. `(==).(1, 2)` or `Base.:(==).(1, 2)`. + quoted = if isa(quoted_func, Expr) && quoted_func.head === :. && length(quoted_func.args) == 1 + Expr(:., quoted_func.args[1], Expr(:tuple, quoted_args...)) + else + Expr(:call, quoted_func, quoted_args...) end if negate @@ -576,14 +585,90 @@ macro test_skip(ex, kws...) return :(record(get_testset(), $testres)) end -function _can_escape_call(@nospecialize ex) - ex.head === :call || return false +function _should_escape_call(@nospecialize ex) + isa(ex, Expr) || return false + + args = if ex.head === :call + ex.args[2:end] + elseif ex.head === :. && length(ex.args) == 2 && isa(ex.args[2], Expr) && ex.args[2].head === :tuple + # Support for broadcasted function calls (e.g. `(==).(1, 2)`) + ex.args[2].args + else + # Expression is not a function call + return false + end + + # Avoid further processing on calls without any arguments + return length(args) > 0 +end + +# Escapes all of the positional arguments and keywords of a function such that we can call +# the function at runtime. +function _escape_call(@nospecialize ex) + if isa(ex, Expr) && ex.head === :call + # Update broadcast comparison calls to the function call syntax + # (e.g. `1 .== 1` becomes `(==).(1, 1)`) + func_str = string(ex.args[1]) + escaped_func = if first(func_str) == '.' + esc(Expr(:., Symbol(func_str[2:end]))) + else + esc(ex.args[1]) + end + quoted_func = QuoteNode(ex.args[1]) + args = ex.args[2:end] + elseif isa(ex, Expr) && ex.head === :. && length(ex.args) == 2 && isa(ex.args[2], Expr) && ex.args[2].head === :tuple + # Support for broadcasted function calls (e.g. `(==).(1, 2)`) + escaped_func = if isa(ex.args[1], Expr) && ex.args[1].head == :. + Expr(:call, Expr(:., :Broadcast, QuoteNode(:BroadcastFunction)), esc(ex.args[1])) + else + Expr(:., esc(ex.args[1])) + end + quoted_func = QuoteNode(Expr(:., ex.args[1])) + args = ex.args[2].args + else + throw(ArgumentError("$ex is not a call expression")) + end + + escaped_args = [] + escaped_kwargs = [] - # Broadcasted functions are not currently supported - first(string(ex.args[1])) != '.' || return false + # Positional arguments and keywords that occur before `;`. Note that the keywords are + # being revised into a form we can splat. + for a in args + if isa(a, Expr) && a.head === :parameters + continue + elseif isa(a, Expr) && a.head === :kw + # Keywords that occur before `;`. Note that the keywords are being revised into + # a form we can splat. + push!(escaped_kwargs, Expr(:call, :(=>), QuoteNode(a.args[1]), esc(a.args[2]))) + elseif isa(a, Expr) && a.head === :... + push!(escaped_args, Expr(:..., esc(a.args[1]))) + else + push!(escaped_args, esc(a)) + end + end - # At least one positional argument or keyword - return length(ex.args) > 1 + # Keywords that occur after ';' + if length(args) > 0 && isa(args[1], Expr) && args[1].head === :parameters + for kw in args[1].args + if isa(kw, Expr) && kw.head === :kw + push!(escaped_kwargs, Expr(:call, :(=>), QuoteNode(kw.args[1]), esc(kw.args[2]))) + elseif isa(kw, Expr) && kw.head === :... + push!(escaped_kwargs, Expr(:..., esc(kw.args[1]))) + elseif isa(kw, Expr) && kw.head === :. + push!(escaped_kwargs, Expr(:call, :(=>), QuoteNode(kw.args[2].value), esc(Expr(:., kw.args[1], QuoteNode(kw.args[2].value))))) + elseif isa(kw, Symbol) + push!(escaped_kwargs, Expr(:call, :(=>), QuoteNode(kw), esc(kw))) + end + end + end + + return (; + func=escaped_func, + args=escaped_args, + kwargs=escaped_kwargs, + quoted_func, + ) end # An internal function, called by the code generated by the @test @@ -613,60 +698,22 @@ function get_test_result(ex, source) ex = Expr(:comparison, ex.args[1], ex.head, ex.args[2]) end if isa(ex, Expr) && ex.head === :comparison - # pass all terms of the comparison to `eval_comparison`, as an Expr + # pass all terms of the comparison to `eval_test_comparison`, as a tuple escaped_terms = [esc(arg) for arg in ex.args] quoted_terms = [QuoteNode(arg) for arg in ex.args] - testret = :(eval_test( - Expr(:comparison, $(escaped_terms...)), + testret = :(eval_test_comparison( + ($(escaped_terms...),), Expr(:comparison, $(quoted_terms...)), $(QuoteNode(source)), $negate, )) - elseif isa(ex, Expr) && _can_escape_call(ex) - escaped_func = esc(ex.args[1]) - quoted_func = QuoteNode(ex.args[1]) - - escaped_args = [] - escaped_kwargs = [] - - # Keywords that occur before `;`. Note that the keywords are being revised into - # a form we can splat. - for a in ex.args[2:end] - if isa(a, Expr) && a.head === :kw - push!(escaped_kwargs, Expr(:call, :(=>), QuoteNode(a.args[1]), esc(a.args[2]))) - end - end - - # Keywords that occur after ';' - parameters_expr = ex.args[2] - if isa(parameters_expr, Expr) && parameters_expr.head === :parameters - for a in parameters_expr.args - if isa(a, Expr) && a.head === :kw - push!(escaped_kwargs, Expr(:call, :(=>), QuoteNode(a.args[1]), esc(a.args[2]))) - elseif isa(a, Expr) && a.head === :... - push!(escaped_kwargs, Expr(:..., esc(a.args[1]))) - elseif isa(a, Expr) && a.head === :. - push!(escaped_kwargs, Expr(:call, :(=>), QuoteNode(a.args[2].value), esc(Expr(:., a.args[1], QuoteNode(a.args[2].value))))) - elseif isa(a, Symbol) - push!(escaped_kwargs, Expr(:call, :(=>), QuoteNode(a), esc(a))) - end - end - end - - # Positional arguments - for a in ex.args[2:end] - isa(a, Expr) && a.head in (:kw, :parameters) && continue - - if isa(a, Expr) && a.head === :... - push!(escaped_args, Expr(:..., esc(a.args[1]))) - else - push!(escaped_args, esc(a)) - end - end - - testret = :(eval_test( - Expr(:call, $escaped_func, Expr(:parameters, $(escaped_kwargs...)), $(escaped_args...)), - Expr(:call, $quoted_func), + elseif _should_escape_call(ex) + call = _escape_call(ex) + testret = :(eval_test_function( + $(call.func), + ($(call.args...),), + ($(call.kwargs...),), + $(call.quoted_func), $(QuoteNode(source)), $negate, )) diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index 5b2963a0e1ea4..d3dae39d2c577 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license using Test, Random -using Test: guardseed, _can_escape_call +using Test: guardseed, _should_escape_call, _escape_call using Serialization using Distributed: RemoteException @@ -126,50 +126,52 @@ let fails = @testset NoThrowTestSet begin @test ==(1 - 2, 2 - 1) # 8 - Fail - splatting @test ==(1:2...) - # 9 - Fail - isequal + # 9 & 10 - Fail - broadcast + @test 1*1 .== 2*2 + @test (==).(1*1, 2*2) + # 11 & 12 - Fail qualified functions + @test Base.:(==)(1*1, 2*2) + @test Base.:(==).(1*1, 2*2) + # 13 - Fail - isequal @test isequal(0 / 0, 1 / 0) - # 10 - Fail - function splatting + # 14 - Fail - function splatting @test isequal(1:2...) - # 11 - Fail - isapprox + # 15 - Fail - isapprox @test isapprox(0 / 1, -1 / 0) - # 12 & 13 - Fail - function with keyword + # 16 & 17 - Fail - function with keyword @test isapprox(1 / 2, 2 / 1, atol=1 / 1) @test isapprox(1 - 2, 2 - 1; atol=1 - 1) - # 14 - Fail - function keyword splatting + # 18 - Fail - function keyword splatting k = [(:atol, 0), (:nans, true)] @test isapprox(1, 2; k...) - # 15 - Fail - call negation + # 19 - Fail - call negation @test !isequal(1, 2 - 1) - # 16 - Fail - comparison negation + # 20 - Fail - comparison negation @test !(2 + 3 == 1 + 4) - # 17 - Fail - chained negation + # 21 - Fail - chained negation @test !(2 + 3 == 1 + 4 == 5) - # 18 - Fail - isempty + # 22 - Fail - isempty nonempty = [1, 2, 3] @test isempty(nonempty) str1 = "Hello" str2 = "World" - # 19 - Fail - occursin + # 23 - Fail - occursin @test occursin(str1, str2) - # 20 - Fail - startswith + # 24 - Fail - startswith @test startswith(str1, str2) - # 21 - Fail - endswith + # 25 - Fail - endswith @test endswith(str1, str2) - # 22 - Fail - contains + # 26 - Fail - contains @test Base.contains(str1, str2) - # 23 - Fail - issetequal - a = [1, 2] - b = [1, 3] - @test issetequal(a, b) - # 24 - Fail - Type Comparison + # 27 - Fail - issetequal + @test issetequal([2, 3] .- 1, [1, 3]) + # 28 - Fail - Type Comparison @test typeof(1) <: typeof("julia") - # 27 - 28 - Fail - wrong message + # 29 - 32 - Fail - wrong message @test_throws "A test" error("a test") @test_throws r"sqrt\([Cc]omplx" sqrt(-1) @test_throws str->occursin("a T", str) error("a test") @test_throws ["BoundsError", "acquire", "1-element", "at index [2]"] [1][2] - # 29 - Fail - broadcast - @test 1 .== 2 end for fail in fails @test fail isa Test.Fail @@ -216,110 +218,125 @@ let fails = @testset NoThrowTestSet begin end let str = sprint(show, fails[9]) + @test occursin("Expression: 1 * 1 .== 2 * 2", str) + @test occursin("Evaluated: 1 .== 4", str) + end + + let str = sprint(show, fails[10]) + @test occursin("Expression: (==).(1 * 1, 2 * 2)", str) + @test occursin("Evaluated: (==).(1, 4)", str) + end + + let str = sprint(show, fails[11]) + @test occursin("Expression: Base.:(==)(1 * 1, 2 * 2)", str) + @test occursin("Evaluated: Base.:(==)(1, 4)", str) + end + + let str = sprint(show, fails[12]) + @test occursin("Expression: Base.:(==).(1 * 1, 2 * 2)", str) + @test occursin("Evaluated: Base.:(==).(1, 4)", str) + end + + let str = sprint(show, fails[13]) @test occursin("Expression: isequal(0 / 0, 1 / 0)", str) @test occursin("Evaluated: isequal(NaN, Inf)", str) end - let str = sprint(show, fails[10]) + let str = sprint(show, fails[14]) @test occursin("Expression: isequal(1:2...)", str) @test occursin("Evaluated: isequal(1, 2)", str) end - let str = sprint(show, fails[11]) + let str = sprint(show, fails[15]) @test occursin("Expression: isapprox(0 / 1, -1 / 0)", str) @test occursin("Evaluated: isapprox(0.0, -Inf)", str) end - let str = sprint(show, fails[12]) + let str = sprint(show, fails[16]) @test occursin("Expression: isapprox(1 / 2, 2 / 1, atol = 1 / 1)", str) @test occursin("Evaluated: isapprox(0.5, 2.0; atol = 1.0)", str) end - let str = sprint(show, fails[13]) + let str = sprint(show, fails[17]) @test occursin("Expression: isapprox(1 - 2, 2 - 1; atol = 1 - 1)", str) @test occursin("Evaluated: isapprox(-1, 1; atol = 0)", str) end - let str = sprint(show, fails[14]) + let str = sprint(show, fails[18]) @test occursin("Expression: isapprox(1, 2; k...)", str) @test occursin("Evaluated: isapprox(1, 2; atol = 0, nans = true)", str) end - let str = sprint(show, fails[15]) + let str = sprint(show, fails[19]) @test occursin("Expression: !(isequal(1, 2 - 1))", str) @test occursin("Evaluated: !(isequal(1, 1))", str) end - let str = sprint(show, fails[16]) + let str = sprint(show, fails[20]) @test occursin("Expression: !(2 + 3 == 1 + 4)", str) @test occursin("Evaluated: !(5 == 5)", str) end - let str = sprint(show, fails[17]) + let str = sprint(show, fails[21]) @test occursin("Expression: !(2 + 3 == 1 + 4 == 5)", str) @test occursin("Evaluated: !(5 == 5 == 5)", str) end - let str = sprint(show, fails[18]) + let str = sprint(show, fails[22]) @test occursin("Expression: isempty(nonempty)", str) @test occursin("Evaluated: isempty([1, 2, 3])", str) end - let str = sprint(show, fails[19]) + let str = sprint(show, fails[23]) @test occursin("Expression: occursin(str1, str2)", str) @test occursin("Evaluated: occursin(\"Hello\", \"World\")", str) end - let str = sprint(show, fails[20]) + let str = sprint(show, fails[24]) @test occursin("Expression: startswith(str1, str2)", str) @test occursin("Evaluated: startswith(\"Hello\", \"World\")", str) end - let str = sprint(show, fails[21]) + let str = sprint(show, fails[25]) @test occursin("Expression: endswith(str1, str2)", str) @test occursin("Evaluated: endswith(\"Hello\", \"World\")", str) end - let str = sprint(show, fails[22]) + let str = sprint(show, fails[26]) @test occursin("Expression: Base.contains(str1, str2)", str) @test occursin("Evaluated: Base.contains(\"Hello\", \"World\")", str) end - let str = sprint(show, fails[23]) - @test occursin("Expression: issetequal(a, b)", str) + let str = sprint(show, fails[27]) + @test occursin("Expression: issetequal([2, 3] .- 1, [1, 3])", str) @test occursin("Evaluated: issetequal([1, 2], [1, 3])", str) end - let str = sprint(show, fails[24]) + let str = sprint(show, fails[28]) @test occursin("Expression: typeof(1) <: typeof(\"julia\")", str) @test occursin("Evaluated: $(typeof(1)) <: $(typeof("julia"))", str) end - let str = sprint(show, fails[25]) + let str = sprint(show, fails[29]) @test occursin("Expected: \"A test\"", str) @test occursin("Message: \"a test\"", str) end - let str = sprint(show, fails[26]) + let str = sprint(show, fails[30]) @test occursin("Expected: r\"sqrt\\([Cc]omplx\"", str) @test occursin(r"Message: .*Try sqrt\(Complex", str) end - let str = sprint(show, fails[27]) + let str = sprint(show, fails[31]) @test occursin("Expected: < match function >", str) @test occursin("Message: \"a test\"", str) end - let str = sprint(show, fails[28]) + let str = sprint(show, fails[32]) @test occursin("Expected: [\"BoundsError\", \"acquire\", \"1-element\", \"at index [2]\"]", str) @test occursin(r"Message: \"BoundsError.* 1-element.*at index \[2\]", str) end - let str = sprint(show, fails[29]) - @test occursin("Expression: 1 .== 2", str) - @test !occursin("Evaluated", str) - end - end struct BadError <: Exception end @@ -1793,13 +1810,52 @@ end end end -@testset "_can_escape_call" begin - @test !_can_escape_call(:(f())) - @test _can_escape_call(:(f(x))) - @test _can_escape_call(:(f(; x))) - @test _can_escape_call(:(f(x=1))) +@testset "_should_escape_call" begin + @test !_should_escape_call(:(f())) + @test _should_escape_call(:(f(x))) + @test _should_escape_call(:(x == y)) + @test _should_escape_call(:(f.(x))) + @test !_should_escape_call(:f) + @test !_should_escape_call(:(f = 1)) + @test !_should_escape_call(:(f.x)) +end + +@testset "_escape_call" begin + @testset "invalid call" begin + @test_throws ArgumentError _escape_call(:f) + @test_throws ArgumentError _escape_call(:(f = 1)) + @test_throws ArgumentError _escape_call(:(f.x)) + end - @test _can_escape_call(:(x == y)) - @test !_can_escape_call(:(x .== y)) - @test !_can_escape_call(:((==).(x, y))) + @testset "positional arguments" begin + func = esc(:f) + quoted_func = :(:f) + @test _escape_call(:(f())) == (; func, args=[], kwargs=[], quoted_func) + @test _escape_call(:(f(x))) == (; func, args=[esc(:x)], kwargs=[], quoted_func) + @test _escape_call(:(f(x...))) == (; func, args=[:($(esc(:x))...)], kwargs=[], quoted_func) + end + + @testset "keyword arguments" begin + func = esc(:f) + quoted_func = :(:f) + @test _escape_call(:(f(y=1))) == (; func, args=[], kwargs=[:(:y => $(esc(1)))], quoted_func) + @test _escape_call(:(f(; y))) == (; func, args=[], kwargs=[:(:y => $(esc(:y)))], quoted_func) + @test _escape_call(:(f(; y=1))) == (; func, args=[], kwargs=[:(:y => $(esc(1)))], quoted_func) + @test _escape_call(:(f(y=1; z))) == (; func, args=[], kwargs=[:(:y => $(esc(1))), :(:z => $(esc(:z)))], quoted_func) + @test _escape_call(:(f(; y.z))) == (; func, args=[], kwargs=[:(:z => $(esc(:(y.z))))], quoted_func) + @test _escape_call(:(f(; y...))) == (; func, args=[], kwargs=[:($(esc(:y))...)], quoted_func) + end + + @testset "comparison" begin + @test _escape_call(:(x == y)) == (; func=esc(:(==)), args=[esc(:x), esc(:y)], kwargs=[], quoted_func=:(:(==))) + end + + @testset "broadcast" begin + args = [esc(:x), esc(:y)] + kwargs = [] + @test _escape_call(:(f.(x, y))) == (; func=Expr(:., esc(:f)), args, kwargs, quoted_func=QuoteNode(Expr(:., :f))) + @test _escape_call(:(Main.f.(x, y))) == (; func=:(Broadcast.BroadcastFunction($(esc(:(Main.f))))), args, kwargs, quoted_func=QuoteNode(Expr(:., :(Main.f)))) + @test _escape_call(:(x .== y)) == (; func=esc(:(.==)), args, kwargs, quoted_func=:(:.==)) + @test _escape_call(:((==).(x, y))) == (; func=Expr(:., esc(:(==))), args, kwargs, quoted_func=QuoteNode(Expr(:., :(==)))) + end end From 694cf5bb5d620b4791d786946f4b54a18d7a1f09 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 21 Mar 2025 15:27:56 -0500 Subject: [PATCH 2/2] Add NEWS entry --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index 352ec6dc8ef0c..2aec4f53588a6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -40,6 +40,8 @@ Standard library changes #### Test +* Test failures when using the `@test` macro now show evaluated arguments for all function calls ([#57825], [#57839]). + #### InteractiveUtils External dependencies