From 90b271661ccb85a50c409811594b5f521df23346 Mon Sep 17 00:00:00 2001
From: Florian Atteneder <florian.atteneder@uni-jena.de>
Date: Tue, 13 Aug 2024 16:38:04 +0000
Subject: [PATCH] tests: add kw only_failed to `runtests` command
(https://git.tpi.uni-jena.de/dg/dg1d.jl/-/merge_requests/205)
---
test/IntegrationTests/src/IntegrationTests.jl | 72 ++++++++++++++++---
1 file changed, 64 insertions(+), 8 deletions(-)
diff --git a/test/IntegrationTests/src/IntegrationTests.jl b/test/IntegrationTests/src/IntegrationTests.jl
index 36929902..247bb977 100644
--- a/test/IntegrationTests/src/IntegrationTests.jl
+++ b/test/IntegrationTests/src/IntegrationTests.jl
@@ -8,6 +8,7 @@ using LinearAlgebra
const REFDIR = abspath(joinpath(@__DIR__, "..", "refs"))
const TESTDIR = abspath(joinpath(@__DIR__, "..", "tests"))
+const FAILED_LOG = joinpath(TESTDIR, "failed.log")
const TERM_GREEN = "\033[32m"
const TERM_BLINK_RED = "\033[5m\033[31m"
const TERM_RESET = "\033[0m"
@@ -352,7 +353,7 @@ end
Run integration tests for which there is reference data
in `refs/` and a configuration entry in `refs/testconfig.toml`.
"""
-function runtests(tests::Vector{String}=String[])
+function impl_runtests(tests::Vector{String}=String[])
### load testconfig and gather files
@@ -537,22 +538,77 @@ Final result: $(final_result ? TERM_GREEN*"PASSED"*TERM_RESET : TERM_B
write(errlogfilename, errlogstr)
end
+ open(FAILED_LOG, "w") do f
+ for i in 1:length(tests)
+ status = runs_status[i]
+ isnothing(status) && continue # passed
+ println(f, tests[i])
+ end
+ end
return final_result
end
-runtests(tests::String...) = runtests(collect(tests))
-function runtests(rgx::Regex)
- testconfig = load_testconfig()
- allreftests = collect(keys(testconfig))
- tests = filter(allreftests) do test
- contains(test, rgx)
+"""
+ runtests(; only_failed=false)
+ runtests(tests::String...; only_failed=false)
+ runtests(tests::Vector{String}; only_failed=false)
+
+Run the list of tests `tests`. These tests must be listed in `testconfig.txt`.
+"""
+runtests(; kwargs...) = runtests(String[]; kwargs...)
+runtests(tests::String...; kwargs...) = runtests(collect(tests); kwargs...)
+function runtests(tests::Vector{String}; only_failed::Bool=false)
+ if only_failed
+ failed_tests = String[]
+ if isfile(FAILED_LOG)
+ open(FAILED_LOG, "r") do f
+ push!(failed_tests, readline(f))
+ end
+ filter!(failed_tests) do test
+ test in tests
+ end
+ end
+ if length(failed_tests) == 0
+ error("No previously failed tests found that match any tests provided in the list: $tests")
+ end
+ end
+ impl_runtests(tests)
+end
+
+"""
+ runtests(rgx::Regex; only_failed::Bool=false)
+
+Run all tests listed in `testconfig.txt` that match against regex `rgx`.
+If `only_failed=true` then `rgx` is matched against the list of tests which failed
+on last run.
+"""
+function runtests(rgx::Regex; only_failed::Bool=false)
+ tests = if only_failed
+ if !isfile(FAILED_LOG)
+ tests = String[]
+ else
+ failed_tests = String[]
+ open(FAILED_LOG, "r") do f
+ push!(failed_tests, readline(f))
+ end
+ filter!(failed_tests) do test
+ !isnothing(match(rgx, test))
+ end
+ failed_tests
+ end
+ else
+ testconfig = load_testconfig()
+ allreftests = collect(keys(testconfig))
+ filter(allreftests) do test
+ contains(test, rgx)
+ end
end
if length(tests) == 0
error("No tests found using regex $rgx. Make sure there are matching output folders in 'refs/' and a configuration entry in 'refs/testconfig.toml'")
end
- return runtests(tests)
+ return impl_runtests(tests)
end
end
--
GitLab