tests: add kw only_failed to `runtests` command...

tests: add kw only_failed to `runtests` command (!205)

test/IntegrationTests/src/IntegrationTests.jl

@@ -8,6 +8,7 @@ using LinearAlgebra
 
 const REFDIR = abspath(joinpath(@__DIR__, "..", "refs"))
 const TESTDIR = abspath(joinpath(@__DIR__, "..", "tests"))
+const FAILED_LOG = joinpath(TESTDIR, "failed.log")
 const TERM_GREEN     = "\033[32m"
 const TERM_BLINK_RED = "\033[5m\033[31m"
 const TERM_RESET     = "\033[0m"
@@ -352,7 +353,7 @@ end
 Run integration tests for which there is reference data
 in `refs/` and a configuration entry in `refs/testconfig.toml`.
 """
-function runtests(tests::Vector{String}=String[])
+function impl_runtests(tests::Vector{String}=String[])
 
     ### load testconfig and gather files
 
@@ -537,22 +538,77 @@ Final result:           $(final_result ? TERM_GREEN*"PASSED"*TERM_RESET : TERM_B
         write(errlogfilename, errlogstr)
     end
 
+    open(FAILED_LOG, "w") do f
+        for i in 1:length(tests)
+            status = runs_status[i]
+            isnothing(status) && continue # passed
+            println(f, tests[i])
+        end
+    end
 
     return final_result
 end
 
-runtests(tests::String...) = runtests(collect(tests))
 
-function runtests(rgx::Regex)
-    testconfig = load_testconfig()
-    allreftests = collect(keys(testconfig))
-    tests = filter(allreftests) do test
-        contains(test, rgx)
+"""
+    runtests(; only_failed=false)
+    runtests(tests::String...; only_failed=false)
+    runtests(tests::Vector{String}; only_failed=false)
+
+Run the list of tests `tests`. These tests must be listed in `testconfig.txt`.
+"""
+runtests(; kwargs...) = runtests(String[]; kwargs...)
+runtests(tests::String...; kwargs...) = runtests(collect(tests); kwargs...)
+function runtests(tests::Vector{String}; only_failed::Bool=false)
+    if only_failed
+        failed_tests = String[]
+        if isfile(FAILED_LOG)
+            open(FAILED_LOG, "r") do f
+                push!(failed_tests, readline(f))
+            end
+            filter!(failed_tests) do test
+                test in tests
+            end
+        end
+        if length(failed_tests) == 0
+            error("No previously failed tests found that match any tests provided in the list: $tests")
+        end
+    end
+    impl_runtests(tests)
+end
+
+"""
+    runtests(rgx::Regex; only_failed::Bool=false)
+
+Run all tests listed in `testconfig.txt` that match against regex `rgx`.
+If `only_failed=true` then `rgx` is matched against the list of tests which failed
+on last run.
+"""
+function runtests(rgx::Regex; only_failed::Bool=false)
+    tests = if only_failed
+        if !isfile(FAILED_LOG)
+            tests = String[]
+        else
+            failed_tests = String[]
+            open(FAILED_LOG, "r") do f
+                push!(failed_tests, readline(f))
+            end
+            filter!(failed_tests) do test
+                !isnothing(match(rgx, test))
+            end
+            failed_tests
+        end
+    else
+        testconfig = load_testconfig()
+        allreftests = collect(keys(testconfig))
+        filter(allreftests) do test
+            contains(test, rgx)
+        end
     end
     if length(tests) == 0
         error("No tests found using regex $rgx. Make sure there are matching output folders in 'refs/' and a configuration entry in 'refs/testconfig.toml'")
     end
-    return runtests(tests)
+    return impl_runtests(tests)
 end
 
 end