summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/Main.hs17
-rw-r--r--src/Run.hs41
-rw-r--r--src/TestMode.hs18
-rw-r--r--test/script/definition.et2
-rw-r--r--test/script/expansion.et2
-rw-r--r--test/script/list.et2
-rw-r--r--test/script/network.et2
-rw-r--r--test/script/output.et4
-rw-r--r--test/script/run.et38
-rw-r--r--test/script/shell.et20
10 files changed, 98 insertions, 48 deletions
diff --git a/src/Main.hs b/src/Main.hs
index 4e73c69..b6c1078 100644
--- a/src/Main.hs
+++ b/src/Main.hs
@@ -30,6 +30,7 @@ data CmdlineOptions = CmdlineOptions
{ optTest :: TestOptions
, optExclude :: [ Text ]
, optVerbose :: Bool
+ , optReport :: Bool
, optColor :: Maybe Bool
, optShowHelp :: Bool
, optShowVersion :: Bool
@@ -42,6 +43,7 @@ defaultCmdlineOptions = CmdlineOptions
{ optTest = defaultTestOptions
, optExclude = []
, optVerbose = False
+ , optReport = False
, optColor = Nothing
, optShowHelp = False
, optShowVersion = False
@@ -95,6 +97,9 @@ options =
, Option [] [ "keep-going" ]
(NoArg $ to $ \opts -> opts { optKeepGoing = True })
"keep going after a failed test"
+ , Option [] [ "report" ]
+ (NoArg $ \opts -> opts { optReport = True, optTest = (optTest opts) { optKeepGoing = True } })
+ "print summary of passing and failing tests (implies --keep-going)"
, Option [] ["wait"]
(NoArg $ to $ \opts -> opts { optWait = True })
"wait at the end of each test"
@@ -208,8 +213,16 @@ main = do
{ optTcpdump = tcpdump
}
- ok <- runTests out topts lmGlobalDefs tests
- when (not ok) exitFailure
+ Report {..} <- runTests out topts lmGlobalDefs tests
+
+ when (optReport opts) $ do
+ putStrLn $ "Total: " <> show reportTotalCount
+ putStrLn $ "Passed: " <> show reportPassedCount
+ putStrLn $ "Failed: " <> show reportFailedCount
+ forM_ reportFailedList $ \tname -> do
+ putStrLn $ T.unpack $ textTestName tname
+
+ when (reportFailedCount > 0) exitFailure
exitOnError :: Either CustomTestError a -> IO a
exitOnError (Left err) = do
diff --git a/src/Run.hs b/src/Run.hs
index a253e51..27dc756 100644
--- a/src/Run.hs
+++ b/src/Run.hs
@@ -1,5 +1,7 @@
module Run (
module Run.Monad,
+ Report(..),
+ TestName, textTestName,
runTests,
runTest,
@@ -58,17 +60,44 @@ import Test
import Test.Builtins
-runTests :: Output -> TestOptions -> GlobalDefs -> [ Test ] -> IO Bool
+data Report = Report
+ { reportTotalCount :: Int
+ , reportPassedCount :: Int
+ , reportSkippedCount :: Int
+ , reportFailedCount :: Int
+ , reportFailedList :: [ TestName ]
+ }
+
+reportPassed :: Report -> Report
+reportPassed r = r
+ { reportTotalCount = reportTotalCount r + 1
+ , reportPassedCount = reportPassedCount r + 1
+ }
+
+reportFailed :: TestName -> Report -> Report
+reportFailed tname r = r
+ { reportTotalCount = reportTotalCount r + 1
+ , reportFailedCount = reportFailedCount r + 1
+ , reportFailedList = tname : reportFailedList r
+ }
+
+runTests :: Output -> TestOptions -> GlobalDefs -> [ Test ] -> IO Report
runTests out opts gdefs tests = do
go $ concat $ replicate (optRepeat opts) tests
where
go (t : ts) = do
runTest out opts gdefs t >>= \case
- True -> go ts
- False
- | optKeepGoing opts -> go ts >> return False
- | otherwise -> return False
- go [] = return True
+ True -> reportPassed <$> go ts
+ False -> reportFailed (testName t) <$> if
+ | optKeepGoing opts -> go ts
+ | otherwise -> go []
+ go [] = return Report
+ { reportTotalCount = 0
+ , reportPassedCount = 0
+ , reportSkippedCount = 0
+ , reportFailedCount = 0
+ , reportFailedList = []
+ }
runTest :: Output -> TestOptions -> GlobalDefs -> Test -> IO Bool
diff --git a/src/TestMode.hs b/src/TestMode.hs
index 90ced30..d865384 100644
--- a/src/TestMode.hs
+++ b/src/TestMode.hs
@@ -78,8 +78,8 @@ getNextTestNumber = do
modify $ \s -> s { tmsNextTestNumber = num + 1 }
return num
-runSingleTest :: Test -> CommandM Bool
-runSingleTest test = do
+runTestsC :: [ Test ] -> CommandM Report
+runTestsC tests = do
out <- asks tmiOutput
num <- getNextTestNumber
Just LoadedModules {..} <- gets tmsModules
@@ -88,11 +88,12 @@ runSingleTest test = do
{ optDefaultTool = fromMaybe "/bin/true" $ configTool =<< mbconfig
, optTestDir = ".test" <> show num
, optKeep = True
+ , optKeepGoing = True
, optHookTestResult = \tname res -> do
flip runReaderT out $ outLine OutputTestRaw Nothing $
"run-test-result " <> testNameBase tname <> " " <> (if res then "done" else "failed")
}
- liftIO (runTest out opts lmGlobalDefs test)
+ liftIO (runTests out opts lmGlobalDefs tests)
newtype CommandM a = CommandM (ReaderT TestModeInput (StateT TestModeState (ExceptT String IO)) a)
@@ -164,6 +165,11 @@ cmdRun = do
case filterTests (cfilter <> pfilter) lm of
Left err -> showError "run-failed" err
Right tests -> do
- forM_ tests $ \test -> do
- runSingleTest test
- cmdOut "run-done"
+ Report {..} <- runTestsC tests
+ cmdOut $ T.unwords
+ [ "run-done"
+ , T.pack (show reportTotalCount)
+ , T.pack (show reportPassedCount)
+ , T.pack (show reportSkippedCount)
+ , T.pack (show reportFailedCount)
+ ]
diff --git a/test/script/definition.et b/test/script/definition.et
index 5490349..31ae21f 100644
--- a/test/script/definition.et
+++ b/test/script/definition.et
@@ -18,4 +18,4 @@ test Definition:
expect /run-test-result Test (.*)/ capture result
guard (result == "done")
expect /(.*)/ capture done
- guard (done == "run-done")
+ guard (done == "run-done 1 1 0 0")
diff --git a/test/script/expansion.et b/test/script/expansion.et
index 86a81dc..85c241d 100644
--- a/test/script/expansion.et
+++ b/test/script/expansion.et
@@ -12,4 +12,4 @@ test VariableExpansion:
for str in [ "1", "1.3", "abc", "abc" ]:
expect /child-stdout p $str/
expect /match p $str/
- expect /run-done/
+ expect /run-done 1 1 0 0/
diff --git a/test/script/list.et b/test/script/list.et
index 4b493a5..7a032cb 100644
--- a/test/script/list.et
+++ b/test/script/list.et
@@ -39,4 +39,4 @@ test ListConcat:
local:
expect /run-test-result Test (.*)/ capture result
guard (result == "done")
- expect /run-done/
+ expect /run-done 1 1 0 0/
diff --git a/test/script/network.et b/test/script/network.et
index 62006ad..2978910 100644
--- a/test/script/network.et
+++ b/test/script/network.et
@@ -15,7 +15,7 @@ def run_test_from using p (file):
send "run Test"
expect /run-test-result Test (.*)/ capture result
guard (result == "done")
- expect /run-done/
+ expect /run-done 1 1 0 0/
test DisconnectNode:
diff --git a/test/script/output.et b/test/script/output.et
index f210490..8fb5616 100644
--- a/test/script/output.et
+++ b/test/script/output.et
@@ -24,7 +24,7 @@ test FlushOutput:
expect /run-test-result Test (.*)/ capture result
guard (result == "failed")
- expect /run-done/
+ expect /run-done 1 0 0 1/
test IgnoreOutput:
spawn as p
@@ -54,4 +54,4 @@ test IgnoreOutput:
expect /run-test-result Test (.*)/ capture result
guard (result == "failed")
- expect /run-done/
+ expect /run-done 1 0 0 1/
diff --git a/test/script/run.et b/test/script/run.et
index 87218c0..d58746a 100644
--- a/test/script/run.et
+++ b/test/script/run.et
@@ -20,14 +20,16 @@ test TrivialRun:
local:
expect /run-test-result AlwaysSucceeds (.*)/ capture result
guard (result == "done")
- expect /run-done/
+ expect /run-done ([0-9]+) ([0-9]+) 0 0/ capture total, passed
+ guard (total == passed)
send "run AlwaysFails"
local:
expect /match-fail .*/
expect /run-test-result AlwaysFails (.*)/ capture result
guard (result == "failed")
- expect /run-done/
+ expect /run-done ([0-9]+) 0 0 ([0-9]+)/ capture total, failed
+ guard (total == failed)
test SimpleRun:
@@ -47,7 +49,7 @@ test SimpleRun:
local:
expect /run-test-result Test (.*)/ capture result
guard (result == "done")
- expect /run-done/
+ expect /run-done 1 1 0 0/
flush
for file in should_fail:
@@ -61,7 +63,7 @@ test SimpleRun:
local:
expect /run-test-result Test (.*)/ capture result
guard (result == "failed")
- expect /run-done/
+ expect /run-done 1 0 0 1/
flush
@@ -88,7 +90,7 @@ test RunConfig:
expect /child-stdout p abcdef/
expect /match p abcdef/
expect /run-test-result ExpectEcho done/
- expect /run-done/
+ expect /run-done 3 2 0 1/
test GetSysInfo:
@@ -107,7 +109,7 @@ test GetSysInfo:
send "load-config"
expect /load-config-done/
send "run SysInfo"
- expect /run-done/
+ expect /run-done 1 1 0 0/
test CallStack:
@@ -126,7 +128,7 @@ test CallStack:
local:
expect /run-test-result AG (.*)/ capture result
guard (result == "failed")
- expect /run-done/
+ expect /run-done 1 0 0 1/
flush
send "run AE"
@@ -139,7 +141,7 @@ test CallStack:
local:
expect /run-test-result AE (.*)/ capture result
guard (result == "failed")
- expect /run-done/
+ expect /run-done 1 0 0 1/
flush
send "run BG"
@@ -153,7 +155,7 @@ test CallStack:
local:
expect /run-test-result BG (.*)/ capture result
guard (result == "failed")
- expect /run-done/
+ expect /run-done 1 0 0 1/
flush
send "run CG"
@@ -169,7 +171,7 @@ test CallStack:
local:
expect /run-test-result CG (.*)/ capture result
guard (result == "failed")
- expect /run-done/
+ expect /run-done 1 0 0 1/
flush
send "run BE"
@@ -184,7 +186,7 @@ test CallStack:
local:
expect /run-test-result BE (.*)/ capture result
guard (result == "failed")
- expect /run-done/
+ expect /run-done 1 0 0 1/
flush
send "run CE"
@@ -201,7 +203,7 @@ test CallStack:
local:
expect /run-test-result CE (.*)/ capture result
guard (result == "failed")
- expect /run-done/
+ expect /run-done 1 0 0 1/
flush
@@ -223,7 +225,7 @@ test RunTag:
guard (result == "done")
local:
expect /run-(.*)/ capture done
- guard (done == "done")
+ guard (done == "done 2 2 0 0")
send "run B C"
local:
@@ -243,7 +245,7 @@ test RunTag:
guard (result == "done")
local:
expect /run-(.*)/ capture done
- guard (done == "done")
+ guard (done == "done 5 5 0 0")
test RunTagExclude:
@@ -264,7 +266,7 @@ test RunTagExclude:
guard (result == "done")
local:
expect /run-(.*)/ capture done
- guard (done == "done")
+ guard (done == "done 2 2 0 0")
send "run T1 B1 A C ^B ^A1"
local:
@@ -281,7 +283,7 @@ test RunTagExclude:
guard (result == "done")
local:
expect /run-(.*)/ capture done
- guard (done == "done")
+ guard (done == "done 4 4 0 0")
test RunExcludeConfig1:
@@ -304,7 +306,7 @@ test RunExcludeConfig1:
expect /run-test-result C2 done/
local:
expect /run-(.*)/ capture done
- guard (done == "done")
+ guard (done == "done 6 6 0 0")
test RunExcludeConfig2:
node n
@@ -324,4 +326,4 @@ test RunExcludeConfig2:
expect /run-test-result B2 done/
local:
expect /run-(.*)/ capture done
- guard (done == "done")
+ guard (done == "done 4 4 0 0")
diff --git a/test/script/shell.et b/test/script/shell.et
index 086d3d3..3384353 100644
--- a/test/script/shell.et
+++ b/test/script/shell.et
@@ -16,7 +16,7 @@ test ShellSpawn:
expect /child-fail sh failed at: .*: false/
expect /child-fail sh exit code: 1/
expect /run-test-result ShellFalse failed/
- expect /run-done/
+ expect /run-done 2 1 0 1/
def expect_next_stdout from p (expected):
@@ -57,7 +57,7 @@ test ShellEcho:
with p:
expect /run-test-result Echo done/
- expect /run-done/
+ expect /run-done 1 1 0 0/
test ShellPipe:
@@ -97,7 +97,7 @@ test ShellPipe:
expect /run-test-result PipeRedirect done/
with p:
- expect /run-done/
+ expect /run-done 3 3 0 0/
test ShellWorkingDirectory:
@@ -136,23 +136,23 @@ test ShellExitOnError:
send "run Default"
expect /child-fail sh failed at: .*error-exit.et:4:9: false/
expect /run-test-result Default failed/
- expect /run-done/
+ expect /run-done 1 0 0 1/
flush
send "run DisabledSuccess"
expect /run-test-result DisabledSuccess done/
- expect /run-done/
+ expect /run-done 1 1 0 0/
flush
send "run DisabledFailure"
expect /run-test-result DisabledFailure failed/
- expect /run-done/
+ expect /run-done 1 0 0 1/
flush
send "run Enabled"
expect /child-fail sh failed at: .*error-exit.et:25:9: false/
expect /run-test-result Enabled failed/
- expect /run-done/
+ expect /run-done 1 0 0 1/
flush
@@ -166,17 +166,17 @@ test ShellLogic:
send "run NegateFalse"
expect /run-test-result NegateFalse done/
- expect /run-done/
+ expect /run-done 1 1 0 0/
flush
send "run NegateTrue"
expect /child-fail sh failed at: .*logic.et:9:9: ! true/
expect /run-test-result NegateTrue failed/
- expect /run-done/
+ expect /run-done 1 0 0 1/
flush
send "run NegateNothing"
expect /child-fail sh failed at: .*logic.et:14:9: !/
expect /run-test-result NegateNothing failed/
- expect /run-done/
+ expect /run-done 1 0 0 1/
flush
generated by cgit v1.2.3 (git 2.25.1) at 2026-08-27 03:22:32 +0000