45169/0001: In the test suite, allow test cases to be marked as expected to fail.

See next commit for a use-case.

ChangeLog

@@ -1,3 +1,9 @@
+2020-01-01  Daniel Shahaf  <danielsh@apache.org>
+
+	* 45169/0001: Test/B01cd.ztst, Test/ztst.zsh,
+	Util/ztst-syntax.vim: In the test suite, allow test cases to
+	be marked as expected to fail.
+
 2019-12-31  Daniel Shahaf  <danielsh@apache.org>
 
 	* users/24582 + users/24583: Test/D04parameter.ztst,

Test/B01cd.ztst

@@ -70,11 +70,13 @@
 # the expected status returned by the code when run, or - if it is
 # irrelevant.  An optional set of single-letter flags follows the status
 # or -.  The following are understood:
-#  . d   Don't diff stdout against the expected stdout.
+#  . d  Don't diff stdout against the expected stdout.
 #   D   Don't diff stderr against the expected stderr.
 #   q   All redirection lines given in the test script (not the lines
 #       actually produced by the test) are subject to ordinary quoted shell
 #       expansion (i.e. not globbing).
+#   f   Test is expected to fail. If the test's exit code, stdout, and stderr
+#       match, report a problem; otherwise, carry on to the next test case.
 # This can be followed by a `:' and a message describing the
 # test, which will be printed if the test fails, along with a
 # description of the failure that occurred.  The `:' and message are

Test/ztst.zsh

@@ -366,6 +366,7 @@ ZTST_test() {
   local last match mbegin mend found substlines
   local diff_out diff_err
   local ZTST_skip
+  integer expected_to_fail
 
   while true; do
     rm -f $ZTST_in $ZTST_out $ZTST_err
@@ -460,8 +461,21 @@ $ZTST_curline"
 	fi
       fi
 
+      if [[ $ZTST_flags = *f* ]]; then
+        expected_to_fail=1
+        ZTST_xfail_diff() { ZTST_diff "$@" > /dev/null }
+        ZTST_diff=ZTST_xfail_diff
+      else
+        expected_to_fail=0
+        ZTST_diff=ZTST_diff
+      fi
+
       # First check we got the right status, if specified.
       if [[ $ZTST_xstatus != - && $ZTST_xstatus != $ZTST_status ]]; then
+        if (( expected_to_fail )); then
+          ZTST_verbose 1 "Test failed, as expected."
+          continue
+        fi
 	ZTST_testfailed "bad status $ZTST_status, expected $ZTST_xstatus from:
 $ZTST_code${$(<$ZTST_terr):+
 Error output:
@@ -480,7 +494,11 @@ $(<$ZTST_terr)"
 	rm -rf $ZTST_out
 	print -r -- "${(e)substlines}" >$ZTST_out
       fi
-      if [[ $ZTST_flags != *d* ]] && ! ZTST_diff $diff_out -u $ZTST_out $ZTST_tout; then
+      if [[ $ZTST_flags != *d* ]] && ! $ZTST_diff $diff_out -u $ZTST_out $ZTST_tout; then
+        if (( expected_to_fail )); then
+          ZTST_verbose 1 "Test failed, as expected."
+          continue
+        fi
 	ZTST_testfailed "output differs from expected as shown above for:
 $ZTST_code${$(<$ZTST_terr):+
 Error output:
@@ -492,11 +510,19 @@ $(<$ZTST_terr)}"
 	rm -rf $ZTST_err
 	print -r -- "${(e)substlines}" >$ZTST_err
       fi
-      if [[ $ZTST_flags != *D* ]] && ! ZTST_diff $diff_err -u $ZTST_err $ZTST_terr; then
+      if [[ $ZTST_flags != *D* ]] && ! $ZTST_diff $diff_err -u $ZTST_err $ZTST_terr; then
+        if (( expected_to_fail )); then
+          ZTST_verbose 1 "Test failed, as expected."
+          continue
+        fi
 	ZTST_testfailed "error output differs from expected as shown above for:
 $ZTST_code"
 	return 1
       fi
+      if (( expected_to_fail )); then
+        ZTST_testfailed "test was expected to fail, but passed."
+        return 1
+      fi
     fi
     ZTST_verbose 1 "Test successful."
     [[ -n $last ]] && break

Util/ztst-syntax.vim

@@ -28,7 +28,7 @@ syn include @zsh                   syntax/zsh.vim
 syn match  ztstPayload             /^\s\+\zs.*/ contains=@zsh
 
 syn match  ztstExitCode            /^\d\+\|^-/                nextgroup=ztstFlags
-syn match  ztstFlags               /[.dDq]*:/       contained nextgroup=ztstTestName contains=ztstColon
+syn match  ztstFlags               /[.dDqf]*:/      contained nextgroup=ztstTestName contains=ztstColon
 syn match  ztstColon               /:/              contained
 syn region ztstTestName            start=// end=/$/ contained