Skip to content

throw if no test items found #216

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nickrobinson251 merged 11 commits into from
Jul 18, 2025
Merged

throw if no test items found #216

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
223efa7
Warn if no test items found
nickrobinson251 Jun 30, 2025
81f6d44
fixup! Warn if no test items found
nickrobinson251 Jun 30, 2025
f7e1cb4
Bump version
nickrobinson251 Jun 30, 2025
4974c8e
Throw rather than warn if no test items found
nickrobinson251 Jun 30, 2025
83f02f6
Delete now impossible empty report test case
nickrobinson251 Jun 30, 2025
472e3f9
fix junit test
nickrobinson251 Jun 30, 2025
fca0da8
Merge branch 'main' into npr-throw-no-tests
nickrobinson251 Jul 7, 2025
cba26ca
Update tests
nickrobinson251 Jul 18, 2025
07333fd
Update testset name
nickrobinson251 Jul 18, 2025
e10058f
Update more tests
nickrobinson251 Jul 18, 2025
2762b86
fixup! Update more tests
nickrobinson251 Jul 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 13 additions & 6 deletions src/ReTestItems.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,14 @@ else # @testset does not yet support `failfast`
CompatDefaultTestSet(a...; failfast::Bool=false, kw...) = DefaultTestSet(a...; kw...)
end

struct NoTestException <: Exception
msg::String
end
# Three argument `showerror` with the `backtrace` keyword to swallow the backtrace,
# like for `Test.TestSetException`.
function Base.showerror(io::IO, exc::NoTestException, bt; backtrace=true)
printstyled(io, exc.msg; color=Base.error_color())
end

# copyied from REPL.jl
function softscope(@nospecialize ex)
Expand Down Expand Up @@ -125,11 +133,11 @@ function _validated_paths(paths, should_throw::Bool)
return filter(paths) do p
if !ispath(p)
msg = "No such path $(repr(p))"
should_throw ? throw(ArgumentError(msg)) : @warn msg
should_throw ? throw(NoTestException(msg)) : @warn msg
return false
elseif !(is_test_file(p) || is_testsetup_file(p)) && isfile(p)
msg = "$(repr(p)) is not a test file"
should_throw ? throw(ArgumentError(msg)) : @warn msg
should_throw ? throw(NoTestException(msg)) : @warn msg
return false
else
return true
Expand Down Expand Up @@ -385,11 +393,10 @@ function _runtests_in_current_env(
ntestitems = length(testitems.testitems)
@info "Finished scanning for test items in $(round(time() - inc_time, digits=2)) seconds."
if ntestitems == 0
@warn "No test items found."
else
@info "Scheduling $ntestitems tests on pid $(Libc.getpid())" *
(nworkers == 0 ? "" : " with $nworkers worker processes and $nworker_threads threads per worker.")
throw(NoTestException("No test items found."))
end
@info "Scheduling $ntestitems tests on pid $(Libc.getpid())" *
(nworkers == 0 ? "" : " with $nworkers worker processes and $nworker_threads threads per worker.")
try
if nworkers == 0
length(cfg.worker_init_expr.args) > 0 && error("worker_init_expr is set, but will not run because number of workers is 0.")
Expand Down
135 changes: 34 additions & 101 deletions test/integrationtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ end
end

@testset "Warn or error when not test file" begin
using ReTestItems: NoTestException
pkg = joinpath(TEST_PKG_DIR, "TestsInSrc.jl")

# warn if the path does not exist
Expand All @@ -95,10 +96,10 @@ end
runtests(dne)
end
# throw if `validate_paths`
@test_throws ArgumentError(dne_msg) runtests(dne; validate_paths=true)
@test_throws NoTestException(dne_msg) runtests(dne; validate_paths=true)
# test setting `validate_paths` via environment variable
withenv("RETESTITEMS_VALIDATE_PATHS" => 1) do
@test_throws ArgumentError(dne_msg) runtests(dne)
@test_throws NoTestException(dne_msg) runtests(dne)
end

# warn if the file is not a test file
Expand All @@ -109,15 +110,15 @@ end
runtests(file)
end
# throw if `validate_paths`
@test_throws ArgumentError(file_msg) runtests(file; validate_paths=true)
@test_throws NoTestException(file_msg) runtests(file; validate_paths=true)

# Warn for each invalid path
@test_logs (:warn, dne_msg) (:warn, file_msg) match_mode=:any begin
runtests(dne, file)
end
# Throw on first invalid path if `validate_paths`
@test_throws ArgumentError(dne_msg) runtests(dne, file; validate_paths=true)
@test_throws ArgumentError(file_msg) runtests(file, dne; validate_paths=true)
@test_throws NoTestException(dne_msg) runtests(dne, file; validate_paths=true)
@test_throws NoTestException(file_msg) runtests(file, dne; validate_paths=true)

# Warn for each invalid path and still run valid ones
test_file = joinpath(pkg, "src", "foo_test.jl")
Expand All @@ -129,7 +130,7 @@ end
end
@test n_tests(results) == 2 # foo_test.jl has 2 tests
# Throw on first invalid path, even if some are valid, if `validate_paths`
@test_throws ArgumentError(dne_msg) runtests(test_file, dne, file; validate_paths=true)
@test_throws NoTestException(dne_msg) runtests(test_file, dne, file; validate_paths=true)
end

@testset "filter `runtests(func, x)`" begin
Expand All @@ -142,10 +143,7 @@ end
@assert n_total > 0

# can exclude everything
results = encased_testset() do
runtests(x->false, pkg)
end
@test n_tests(results) == 0
@test_throws ReTestItems.NoTestException runtests(x->false, pkg)

# there is a `@testitem "bar"` -- filter to just that testitem.
results = encased_testset() do
Expand All @@ -154,10 +152,7 @@ end
@test n_passed(results) > 0
@test n_tests(results) < n_total

results = encased_testset() do
runtests(ti -> contains(ti.name, "bar_"), pkg)
end
@test n_tests(results) == 0
@test_throws ReTestItems.NoTestException runtests(ti -> contains(ti.name, "bar_"), pkg)

# there is a `@testitem "b"` tagged `:b_tag` -- filter to just that testitem.
results = encased_testset() do
Expand All @@ -173,40 +168,9 @@ end
@test n_passed(results) == 1
@test n_tests(results) == 1

results = encased_testset() do
runtests(ti -> :b_tag in ti.tags, pkg; name="b", tags=:nope)
end
@test n_passed(results) == 0
@test n_tests(results) == 0
@test_throws ReTestItems.NoTestException runtests(ti -> :b_tag in ti.tags, pkg; name="nope")

results = encased_testset() do
runtests(ti -> :b_tag in ti.tags, pkg; name="nope")
end
@test n_passed(results) == 0
@test n_tests(results) == 0

## TODO: Are we okay to remove these tests?
## passing a `shouldrun` function as the first arg has never been documented, and
## when it has come up as a workaround for people, we have only ever said you can filter
## on `ti.name` and `ti.tags`, so i think it is okay to remove these tests that use
## `ti.file` (and not support filtering on `ti.file)
##
# # there is a `bar_test.jl` -- filter to just that file.
# results = encased_testset() do
# runtests(ti -> contains(ti.file, "bar_"), pkg)
# end
# @test n_passed(results) > 0
# @test n_tests(results) < n_total

# # test we can filter by directory (all tests are in `src/`)
# results_test_dir = encased_testset() do
# runtests(ti -> startswith(ti.file, "$pkg/test"), pkg)
# end
# results_src_dir = encased_testset() do
# runtests(ti -> startswith(ti.file, "$pkg/src"), pkg)
# end
# @test n_tests(results_src_dir) == n_total
# @test n_tests(results_test_dir) == 0
@test_throws ReTestItems.NoTestException runtests(ti -> :b_tag in ti.tags, pkg; name="b", tags=[:nope])

# can only filter on `ti.name` and `ti.tags` (at least for now)
@test_throws "no field file" runtests(ti -> contains(ti.file, "bar_"), pkg)
Expand Down Expand Up @@ -515,14 +479,11 @@ end
@test n_tests(results) == 1

# There is no test with tag3
results = encased_testset(()->runtests(file, tags=[:tag1, :tag3]))
@test n_tests(results) == 0
@test_throws ReTestItems.NoTestException runtests(file, tags=[:tag1, :tag3])

results = encased_testset(()->runtests(file, tags=[:tag3]))
@test n_tests(results) == 0
@test_throws ReTestItems.NoTestException runtests(file, tags=[:tag3])

results = encased_testset(()->runtests(file, tags=:tag3))
@test n_tests(results) == 0
@test_throws ReTestItems.NoTestException runtests(file, tags=:tag3)
end

@testset "filter `runtests(x; name)`" begin
Expand All @@ -534,17 +495,15 @@ end
results = encased_testset(()->runtests(file))
@assert n_tests(results) == 3

results = encased_testset(()->runtests(file, name=""))
@test n_tests(results) == 0
@test_throws ReTestItems.NoTestException runtests(file, name="")

results = encased_testset(()->runtests(file, name="Test item no tags"))
@test n_tests(results) == 1

results = encased_testset(()->runtests(file, name=@view "Test item no tags"[begin:end]))
@test n_tests(results) == 1

results = encased_testset(()->runtests(file, name=r"No such name in that file"))
@test n_tests(results) == 0
@test_throws ReTestItems.NoTestException runtests(file, name=r"No such name in that file")

results = encased_testset(()->runtests(file, name=r"Test item"))
@test n_tests(results) == 3
Expand All @@ -566,14 +525,11 @@ end
@assert n_tests(results) == 3


results = encased_testset(()->runtests(file, name="", tags=Symbol[]))
@test n_tests(results) == 0
@test_throws ReTestItems.NoTestException runtests(file, name="", tags=Symbol[])

results = encased_testset(()->runtests(file, name=r".", tags=:tag3))
@test n_tests(results) == 0
@test_throws ReTestItems.NoTestException runtests(file, name=r".", tags=:tag3)

results = encased_testset(()->runtests(file, name="", tags=:tag3))
@test n_tests(results) == 0
@test_throws ReTestItems.NoTestException runtests(file, name="", tags=:tag3)

results = encased_testset(()->runtests(file, name=r".", tags=Symbol[]))
@test n_tests(results) == 3
Expand All @@ -597,26 +553,12 @@ end
results = encased_testset(()->runtests(file))
@assert n_tests(results) == 3

results = encased_testset(()->runtests(ti-> false, file, name="", tags=:tag3))
@test n_tests(results) == 0

results = encased_testset(()->runtests(ti-> false, file, name="", tags=Symbol[]))
@test n_tests(results) == 0

results = encased_testset(()->runtests(ti-> false, file, name=r".", tags=:tag3))
@test n_tests(results) == 0

results = encased_testset(()->runtests(ti-> true, file, name="", tags=:tag3))
@test n_tests(results) == 0

results = encased_testset(()->runtests(ti-> true, file, name="", tags=Symbol[]))
@test n_tests(results) == 0

results = encased_testset(()->runtests(ti-> true, file, name=r".", tags=:tag3))
@test n_tests(results) == 0

results = encased_testset(()->runtests(ti-> false, file, name=r".", tags=Symbol[]))
@test n_tests(results) == 0
@test_throws ReTestItems.NoTestException runtests(ti-> false, file, name="", tags=:tag3)
@test_throws ReTestItems.NoTestException runtests(ti-> false, file, name=r".", tags=:tag3)
@test_throws ReTestItems.NoTestException runtests(ti-> true, file, name="", tags=:tag3)
@test_throws ReTestItems.NoTestException runtests(ti-> true, file, name="", tags=Symbol[])
@test_throws ReTestItems.NoTestException runtests(ti-> true, file, name=r".", tags=:tag3)
@test_throws ReTestItems.NoTestException runtests(ti-> false, file, name=r".", tags=Symbol[])

results = encased_testset(()->runtests(ti-> true, file, name=r".", tags=Symbol[]))
@test n_tests(results) == 3
Expand Down Expand Up @@ -829,8 +771,7 @@ end
@testset "`runtests` finds no testitems" begin
file = joinpath(TEST_FILES_DIR, "_empty_file_test.jl")
for nworkers in (0, 1)
results = encased_testset(()->runtests(file; nworkers))
@test n_tests(results) == 0
@test_throws ReTestItems.NoTestException runtests(file; nworkers)
end
end

Expand Down Expand Up @@ -914,8 +855,7 @@ end
@test n_tests(results) == 2
results = encased_testset(() -> runtests(file; tags=[:xyz]))
@test n_tests(results) == 1
results = encased_testset(() -> runtests(filter_func, file))
@test n_tests(results) == 0
@test_throws ReTestItems.NoTestException runtests(filter_func, file)
end
end

Expand Down Expand Up @@ -1547,20 +1487,13 @@ end
@test_throws "`test_end_expr` must be a `:block` expression" runtests(; test_end_expr=:(@assert false))
end

@testset "warn if no test items" begin
msg = "No test items found."
@test_logs (:warn, msg) match_mode=:any begin
runtests(joinpath(TEST_FILES_DIR, "_empty_file_test.jl"))
end
@test_logs (:warn, msg) match_mode=:any begin
runtests(joinpath(TEST_FILES_DIR, "_empty_file_test.jl"); nworkers=1)
end
@test_logs (:warn, msg) match_mode=:any begin
runtests(joinpath(TEST_FILES_DIR, "_happy_tests.jl"); name="blahahahaha_nope")
end
@test_logs (:warn, msg) match_mode=:any begin
runtests(joinpath(TEST_FILES_DIR, "_happy_tests.jl"); tags=[:blahahahaha_nope])
end
@testset "throw if no test items" begin
using ReTestItems: NoTestException
exc = NoTestException("No test items found.")
@test_throws exc runtests(joinpath(TEST_FILES_DIR, "_empty_file_test.jl"))
@test_throws exc runtests(joinpath(TEST_FILES_DIR, "_empty_file_test.jl"); nworkers=1)
@test_throws exc runtests(joinpath(TEST_FILES_DIR, "_happy_tests.jl"); name="blahahahaha_nope")
@test_throws exc runtests(joinpath(TEST_FILES_DIR, "_happy_tests.jl"); tags=[:blahahahaha_nope])
end

end # integrationtests.jl testset
6 changes: 3 additions & 3 deletions test/internals.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,22 +264,22 @@ end
@assert !ispath("foo")
@test _validated_paths(("foo",), false) == ()
@test_logs (:warn, "No such path \"foo\"") _validated_paths(("foo",), false)
@test_throws ArgumentError("No such path \"foo\"") _validated_paths(("foo",), true)
@test_throws ReTestItems.NoTestException("No such path \"foo\"") _validated_paths(("foo",), true)

@assert isfile(test_file)
@assert !ispath("foo")
paths = (test_file, "foo",)
@test _validated_paths(paths, false) == (test_file,)
@test_logs (:warn, "No such path \"foo\"") _validated_paths(paths, false)
@test_throws ArgumentError("No such path \"foo\"") _validated_paths(paths, true)
@test_throws ReTestItems.NoTestException("No such path \"foo\"") _validated_paths(paths, true)

nontest_file = joinpath(testfiles_dir, "_empty_file.jl")
@assert isfile(nontest_file)
@assert !ReTestItems.is_test_file(nontest_file)
@assert !ReTestItems.is_testsetup_file(nontest_file)
@test _validated_paths((nontest_file,), false) == ()
@test_logs (:warn, "$(repr(nontest_file)) is not a test file") _validated_paths((nontest_file,), false)
@test_throws ArgumentError("$(repr(nontest_file)) is not a test file") _validated_paths((nontest_file,), true)
@test_throws ReTestItems.NoTestException("$(repr(nontest_file)) is not a test file") _validated_paths((nontest_file,), true)
end

@testset "skiptestitem" begin
Expand Down
17 changes: 1 addition & 16 deletions test/junit_xml.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,26 +142,11 @@ end
end
end

@testset "JUnit empty report" begin
empty_report = strip("""
<?xml version="1.0" encoding="UTF-8"?>
<testsuites timestamp="" time="0.0" tests="0" skipped="0" failures="0" errors="0">
</testsuites>
""")
mktempdir() do dir
withenv("RETESTITEMS_REPORT_LOCATION" => dir) do
runtests("testfiles/_empty_file_test.jl"; report=true)
end
report = read(only(filter(endswith("xml"), readdir(dir, join=true))), String)
@test report == empty_report
end
end

@testset "Respect `RETESTITEMS_REPORT`" begin
for report in (true, false)
mktempdir() do dir
withenv("RETESTITEMS_REPORT" => report, "RETESTITEMS_REPORT_LOCATION" => dir) do
runtests("testfiles/_empty_file_test.jl")
runtests("testfiles/_happy_tests.jl"; name="happy 1")
end
n_reports = length(filter(endswith("xml"), readdir(dir, join=true)))
if report
Expand Down
Loading