Merge remote-tracking branch 'origin/5.9'

* origin/5.9:
  unposted: Move a new incompatibility notice.
  unposted: Fix trailing whitespace in test expectations.
  45342: Add tests for interaction between autoloadable parameters and module loading.
  45313: _git: Support completion from outside of a worktree when --git-dir/--work-tree are specified on the command line
  45304: Do execute the always block even when the try/always block itself is the last command.
  45292: D02glob: Add regression test for macOS stat(2) misbehaviour
  45291: A glob with a trailing slash will now match unreadable/unexecutable directories.
  45288: _git: Complete bisect/new as well as bisect/bad.
  45246: Make --disable-multibyte warn, since the test suite fails in that configuration.
  45213: Make --enable-gdbm default to false, rather than default to true with an unavoidable warning.
  unposted (follow-up to 45131): Extra testing by Mikael
  45137: zformat: Allow the specifying minimum width and a dot with an empty maximum width.
  45138: Add zformat unit tests.
  45131: Make a function that redefines itself preserve its tracedness.

ChangeLog

@@ -1,3 +1,64 @@
+2020-03-07  Daniel Shahaf  <danielsh@apache.org>
+
+	* unposted: README: Move a new incompatibility notice.
+
+	* unposted: Test/V01zmodload.ztst: Fix trailing whitespace in
+	test expectations.
+
+2020-01-29  Daniel Shahaf  <danielsh@apache.org>
+
+	* 45342: Test/V01zmodload.ztst: Add tests for interaction
+	between autoloadable parameters and module loading.
+
+2020-01-17  Daniel Shahaf  <danielsh@apache.org>
+
+	* 45313: Completion/Unix/Command/_git: Support completion from
+	outside of a worktree when --git-dir/--work-tree are specified
+	on the command line
+
+2020-01-16  Daniel Shahaf  <danielsh@apache.org>
+
+	* 45304: Src/loop.c, Test/A01grammar.ztst: Do execute the always
+	block even when the try/always block itself is the last command.
+
+2020-01-15  dana  <dana@dana.is>
+
+	* 45292: Test/D02glob.ztst: Add regression test for macOS stat(2)
+	misbehaviour, now worked around by previous fix (see workers/42891)
+
+	* Daniel: 45291: Src/glob.c, Test/D02glob.ztst: A glob with a
+	trailing slash will now match unreadable/unexecutable directories.
+
+2020-01-15  Daniel Shahaf  <danielsh@apache.org>
+
+	* 45288: Completion/Unix/Command/_git: Complete bisect/new as
+	well as bisect/bad.
+
+2020-01-06  Daniel Shahaf  <d.s@daniel.shahaf.name>
+
+	* 45246: configure.ac: Make --disable-multibyte warn, since
+	the test suite fails in that configuration.
+
+2020-01-06  Daniel Shahaf  <danielsh@apache.org>
+
+	* 45213: README, configure.ac: Make --enable-gdbm default to
+	false, rather than default to true with an unavoidable warning.
+
+2019-12-26  Daniel Shahaf  <danielsh@apache.org>
+
+	* unposted (follow-up to 45131): Test/E02xtrace.ztst: Extra
+	testing by Mikael
+
+	* 45137: Src/Modules/zutil.c, Test/V13zformat.ztst: zformat:
+	Allow the specifying minimum width and a dot with an empty
+	maximum width.
+
+	* 45138: Src/Modules/zutil.c, Test/V13zformat.ztst: Add zformat
+	unit tests.
+
+	* 45131: Src/exec.c, Test/E02xtrace.ztst: Make a function that
+	redefines itself preserve its tracedness.
+
 2020-03-06  Daniel Shahaf  <danielsh@apache.org>
 
 	* unposted: Completion/BSD/Type/_obsd_architectures: Remove

Completion/Unix/Command/_git

@@ -6716,14 +6716,12 @@ __git_heads_local () {
   local f gitdir
   declare -a heads
 
-  heads=(${(f)"$(_call_program headrefs git for-each-ref --format='"%(refname:short)"' refs/heads 2>/dev/null)"})
+  heads=(${(f)"$(_call_program headrefs git for-each-ref --format='"%(refname:short)"' refs/heads refs/bisect refs/stash 2>/dev/null)"})
   gitdir=$(_call_program gitdir git rev-parse --git-dir 2>/dev/null)
   if __git_command_successful $pipestatus; then
     for f in HEAD FETCH_HEAD ORIG_HEAD MERGE_HEAD; do
       [[ -f $gitdir/$f ]] && heads+=$f
     done
-    [[ -f $gitdir/refs/stash ]] && heads+=stash
-    [[ -f $gitdir/refs/bisect/bad ]] && heads+=bisect/bad
   fi
 
   __git_describe_commit heads heads-local "local head" "$@"
@@ -7084,21 +7082,23 @@ __git_files_relative () {
 
 (( $+functions[__git_files] )) ||
 __git_files () {
-  local compadd_opts opts tag description gitcdup gitprefix files expl
+  local compadd_opts opts tag description gittoplevel gitprefix files expl
 
   zparseopts -D -E -a compadd_opts V+: J+: 1 2 o+: n f x+: X+: M+: P: S: r: R: q F:
   zparseopts -D -E -a opts -- -cached -deleted -modified -others -ignored -unmerged -killed x+: --exclude+:
   tag=$1 description=$2; shift 2
 
-  gitcdup=$(_call_program gitcdup git rev-parse --show-cdup 2>/dev/null)
+  gittoplevel=$(_call_program toplevel git rev-parse --show-toplevel 2>/dev/null)
   __git_command_successful $pipestatus || return 1
+  [[ -n $gittoplevel ]] && gittoplevel+="/"
 
   gitprefix=$(_call_program gitprefix git rev-parse --show-prefix 2>/dev/null)
   __git_command_successful $pipestatus || return 1
 
   # TODO: --directory should probably be added to $opts when --others is given.
 
-  local pref=$gitcdup$gitprefix$PREFIX
+  local pref=${(Q)${~PREFIX}}
+  [[ $pref[1] == '/' ]] || pref=$gittoplevel$gitprefix$pref
 
   # First allow ls-files to pattern-match in case of remote repository
   files=(${(0)"$(_call_program files git ls-files -z --exclude-standard ${(q)opts} -- ${(q)${pref:+$pref\*}:-.} 2>/dev/null)"})
@@ -8140,7 +8140,8 @@ _git() {
         ;;
       (option-or-argument)
         curcontext=${curcontext%:*:*}:git-$words[1]:
-	(( $+opt_args[--git-dir] )) && local -x GIT_DIR=$opt_args[--git-dir]
+        (( $+opt_args[--git-dir] )) && local -x GIT_DIR=${(Q)${~opt_args[--git-dir]}}
+        (( $+opt_args[--work-tree] )) && local -x GIT_WORK_TREE=${(Q)${~opt_args[--work-tree]}}
 	if ! _call_function ret _git-$words[1]; then
 	  if [[ $words[1] = \!* ]]; then
 	    words[1]=${words[1]##\!}

README

@@ -31,6 +31,13 @@ Zsh is a shell with lots of features.  For a list of some of these, see the
 file FEATURES, and for the latest changes see NEWS.  For more
 details, see the documentation.
 
+Incompatibilities since 5.8
+---------------------------
+
+Build-time change: The default value of the --enable-gdbm configure
+argument has changed from "yes" to "no".  Thus, the zsh/db/gdbm module will
+not be built unless --enable-gdbm is passed explicitly.
+
 Incompatibilities since 5.7.1
 -----------------------------
 

Src/Modules/zutil.c

@@ -797,8 +797,7 @@ static char *zformat_substring(char* instr, char **specs, char **outp,
 	    if ((*s == '.' || testit) && idigit(s[1])) {
 		for (max = 0, s++; idigit(*s); s++)
 		    max = (max * 10) + (int) STOUC(*s) - '0';
-	    }
+	    } else if (*s == '.' || testit)
-	    else if (testit)
 		s++;
 
 	    if (testit && STOUC(*s)) {
@@ -913,13 +912,13 @@ bin_zformat(char *nam, char **args, UNUSED(Options ops), UNUSED(int func))
     switch (opt) {
     case 'f':
 	{
-	    char **ap, *specs[256], *out;
+	    char **ap, *specs[256] = {0}, *out;
 	    int olen, oused = 0;
 
-	    memset(specs, 0, 256 * sizeof(char *));
-
 	    specs['%'] = "%";
 	    specs[')'] = ")";
+
+	    /* Parse the specs in argv. */
 	    for (ap = args + 2; *ap; ap++) {
 		if (!ap[0][0] || ap[0][0] == '-' || ap[0][0] == '.' ||
 		    idigit(ap[0][0]) || ap[0][1] != ':') {

Src/exec.c

@@ -5318,6 +5318,12 @@ execfuncdef(Estate state, Eprog redir_prog)
 		 */
 		removetrapnode(signum);
 	    }
+	    /* Is this function traced and redefining itself? */
+	    if (funcstack && funcstack->tp == FS_FUNC &&
+		    !strcmp(s, funcstack->name)) {
+		Shfunc old = ((Shfunc)shfunctab->getnode(shfunctab, s));
+		shf->node.flags |= old->node.flags & (PM_TAGGED|PM_TAGGED_LOCAL);
+	    }
 	    shfunctab->addnode(shfunctab, ztrdup(s), shf);
 	}
     }

Src/glob.c

@@ -279,11 +279,11 @@ addpath(char *s, int l)
  * foo/ can be used to reference a non-directory foo.  Returns nonzero if   *
  * the file does not exists.                                                */
 
-/**/
 static int
 statfullpath(const char *s, struct stat *st, int l)
 {
     char buf[PATH_MAX+1];
+    int check_for_being_a_directory = 0;
 
     DPUTS(strlen(s) + !*s + pathpos - pathbufcwd >= PATH_MAX,
 	  "BUG: statfullpath(): pathname too long");
@@ -294,17 +294,45 @@ statfullpath(const char *s, struct stat *st, int l)
 	 * Don't add the '.' if the path so far is empty, since
 	 * then we get bogus empty strings inserted as files.
 	 */
+	if (st) {
 	    buf[pathpos - pathbufcwd] = '.';
 	    buf[pathpos - pathbufcwd + 1] = '\0';
 	    l = 0;
 	}
-    unmetafy(buf, NULL);
+	else {
-    if (!st) {
+	    check_for_being_a_directory = 1;
-	char lbuf[1];
-	return access(buf, F_OK) && (!l || readlink(buf, lbuf, 1) < 0);
 	}
+    }
+    unmetafy(buf, NULL);
+    if (st) {
 	return l ? lstat(buf, st) : stat(buf, st);
     }
+    else if (check_for_being_a_directory) {
+	struct stat tmp;
+	if (stat(buf, &tmp))
+	    return -1;
+
+	return S_ISDIR(tmp.st_mode) ? 0 : -1;
+    }
+    else {
+	char lbuf[1];
+
+	/* If it exists, signal success. */
+	if (access(buf, F_OK) == 0)
+	    return 0;
+
+	/* Would a dangling symlink be good enough? */
+	if (l == 0)
+	    return -1;
+
+	/* Is it a dangling symlink? */
+	if (readlink(buf, lbuf, 1) >= 0)
+	    return 0;
+
+	/* Guess it doesn't exist, then. */
+	return -1;
+    }
+}
 
 /* This may be set by qualifier functions to an array of strings to insert
  * into the list instead of the original string. */
@@ -327,11 +355,13 @@ insert(char *s, int checked)
     if (gf_listtypes || gf_markdirs) {
 	/* Add the type marker to the end of the filename */
 	mode_t mode;
-	checked = statted = 1;
 	if (statfullpath(s, &buf, 1)) {
 	    unqueue_signals();
 	    return;
 	}
+	else {
+	    checked = statted = 1;
+	}
 	mode = buf.st_mode;
 	if (gf_follow) {
 	    if (!S_ISLNK(mode) || statfullpath(s, &buf2, 0))
@@ -387,11 +417,10 @@ insert(char *s, int checked)
 	    qn = qn->next;
 	}
     } else if (!checked) {
-	if (statfullpath(s, &buf, 1)) {
+	if (statfullpath(s, NULL, 1)) {
 	    unqueue_signals();
 	    return;
 	}
-	statted = 1;
 	news = dyncat(pathbuf, news);
     } else
 	news = dyncat(pathbuf, news);

Src/loop.c

@@ -742,7 +742,7 @@ exectry(Estate state, int do_exec)
 
     /* The :try clause */
     ++try_tryflag;
-    execlist(state, 1, do_exec);
+    execlist(state, 1, 0);
     --try_tryflag;
 
     /* Don't record errflag here, may be reset.  However, */

Test/A01grammar.ztst

@@ -930,5 +930,5 @@ F:Note that the behaviour of 'exit' inside try-list inside a function is unspeci
 >or false
 
  $ZTST_testdir/../Src/zsh -fc '{ ( ) } always { echo foo }'
--f:exec last command optimization inhibited for try/always
+0:exec last command optimization inhibited for try/always
 >foo

Test/D02glob.ztst

@@ -729,16 +729,12 @@
 >not/exist
 >exist
 
- if [[ $OSTYPE = cygwin* ]]; then
-   ZTST_skip='Cygwin ignores search permission of directories'
- else
  mkdir -m 000 glob.tmp/secret-d000
  mkdir -m 111 glob.tmp/secret-d111
  mkdir -m 444 glob.tmp/secret-d444
  for 1 in 000 111 444 ; do ln -s secret-d$1 glob.tmp/secret-s$1; done
  print -rC 2 -- glob.tmp/secret-*/ glob.tmp/secret-*(-/)
- fi
+0:unreadable directories can be globbed (users/24619, users/24626)
--f:unreadable directories can be globbed (users/24619, users/24626)
 >glob.tmp/secret-d000/  glob.tmp/secret-d000
 >glob.tmp/secret-d111/  glob.tmp/secret-d111
 >glob.tmp/secret-d444/  glob.tmp/secret-d444
@@ -746,6 +742,17 @@
 >glob.tmp/secret-s111/  glob.tmp/secret-s111
 >glob.tmp/secret-s444/  glob.tmp/secret-s444
 
+ # On macOS, stat(2) allows files to be treated as directories if the calling
+ # process has super-user privileges. e.g., stat() on /my/regular/file/. will
+ # succeed as root but (correctly) fail otherwise. This can produce strange
+ # results when globbing, depending on how it's implemented. This test should,
+ # when run with privileges, confirm that the implementation avoids this
+ # problem. See workers/42891 and workers/45291
+ : > glob.tmp/not-a-directory
+ print -r - glob.tmp/not-a-dir*(N) , glob.tmp/not-a-dir*/(N)
+0:non-directories not globbed as directories
+>glob.tmp/not-a-directory ,
+
  () { echo $1:P } ////dev
 -f:(workers/45367) modifier ':P' squashes multiple slashes
 >/dev  

Test/E02xtrace.ztst

@@ -146,3 +146,37 @@
 ?+(anon):0> '(anon)'
 ?+(anon):0> true
 ?+fn:0> gn
+
+  test_cases=(
+      f            # baseline
+      foo-bar      # Dash
+      ヌ           # Meta (the UTF-8 representation of this character has an 0x83 byte)
+      \$\'ba\\0z\' # Nul, escaped as though by ${(qqqq)}
+  )
+  for 1 in "$test_cases[@]"; do
+    eval "
+      ${1}() {
+        ${1}() { echo inner }
+      }
+      functions -T ${1}
+      ${1}
+      which ${1}
+    "
+  done
+0:a function that redefines itself preserves tracing
+>f () {
+>	# traced
+>	echo inner
+>}
+>foo-bar () {
+>	# traced
+>	echo inner
+>}
+>$'\M-c\M-\C-C\M-\C-L' () {
+>	# traced
+>	echo inner
+>}
+>$'ba\C-@z' () {
+>	# traced
+>	echo inner
+>}

Test/V01zmodload.ztst

@@ -348,6 +348,47 @@
 ?(eval):6: unknown function: systell
 ?(eval):9: file descriptor out of range
 
+ $ZTST_testdir/../Src/zsh -fc '
+   if zmodload -e zsh/parameter; then zmodload -u zsh/parameter; fi
+   unset options
+   zmodload zsh/parameter
+   echo $+options
+ '
+-f:can unset a non-readonly autoloadable parameter before loading the module
+>0
+# Currently prints '1'.
+
+ $ZTST_testdir/../Src/zsh -fc '
+   zmodload zsh/parameter
+   unset options
+   echo $+options
+ '
+0:can unset a non-readonly autoloadable parameter after loading the module
+>0
+
+ $ZTST_testdir/../Src/zsh -fc '
+   if zmodload -e zsh/parameter; then zmodload -u zsh/parameter; fi
+   unset builtins
+ '
+-f:can't unset a readonly autoloadable parameter before loading the module
+*?zsh:?: read-only variable: builtins
+# Currently, the 'unset' succeeds.
+
+ $ZTST_testdir/../Src/zsh -fc '
+   zmodload zsh/parameter
+   unset builtins
+ '
+1:can't unset a readonly autoloadable parameter after loading the module
+?zsh:3: read-only variable: builtins
+
+ $ZTST_testdir/../Src/zsh -fc '
+   zmodload zsh/parameter
+   zmodload -u zsh/parameter
+   echo $options
+ '
+0:unloading a module doesn't implicitly unset autoloadable parameters
+*>(on|off) *
+
  $ZTST_testdir/../Src/zsh -fc "
   MODULE_PATH=${(q)MODULE_PATH}
   #

Test/V13zformat.ztst

@@ -0,0 +1,67 @@
+# Test the use of zformat, if the zsh/zutil module is available.
+
+%prep
+
+  if ! zmodload zsh/zutil 2>/dev/null; then
+    ZTST_unimplemented="can't load the zsh/zutil module for testing"
+  fi
+
+  # Helper function.  Expects a single format using %s and a value for it.
+  zformat_and_print_s() {
+    zformat -f REPLY "$1" "s:$2"
+    print -r - ${(qq)REPLY}
+  }
+
+%test
+
+ zformat_and_print_s '%s'   foo
+ zformat_and_print_s '%5s'  min
+ zformat_and_print_s '%-5s' neg
+ zformat_and_print_s '%5.s' empty
+ zformat_and_print_s '%.5s' max
+ zformat_and_print_s '%.5s' truncated
+0:basic zformat test
+>'foo'
+>'min  '
+>'  neg'
+>'empty'
+>'max'
+>'trunc'
+
+ # There may be a set of digits either before or after the opening parenthesis.
+ zformat_and_print_s 'The answer is "%3(s.yes.no)".' 3
+ zformat_and_print_s 'The answer is "%(3s.yes.no)".' 3
+ # The test number defaults to zero.
+ zformat_and_print_s '%(s.equal.unequal)' -1
+ zformat_and_print_s '%(s.equal.unequal)' 0
+ zformat_and_print_s '%(s.equal.unequal)' 1
+ # Negative numbers are allowed
+ # The delimiter is arbitrary
+ zformat_and_print_s '%-4(s.minus four.)' -4
+ zformat_and_print_s '%(-4s//minus four)' -4
+ # The argument is evaluated as a math expression
+ zformat_and_print_s '%18(s.math.)' '6*3'
+0:basic conditionals test
+>'The answer is "yes".'
+>'The answer is "yes".'
+>'unequal'
+>'equal'
+>'unequal'
+>'minus four'
+>''
+>'math'
+
+ () {
+   zformat -f 1 '%(8n.%(5j.yes.no).no)' 'n:8' 'j:5'
+   echo $1
+ }
+0:nested conditionals test
+>yes
+
+ zformat -a argv . foo:lorem ipsum:bar bazbaz '\\esc\:ape'
+ print -rl -- "$@"
+0:basic -a test
+>foo  .lorem
+>ipsum.bar
+>bazbaz
+>\esc:ape

configure.ac

@@ -444,9 +444,10 @@ dnl Do you want to look for capability support?
 AC_ARG_ENABLE(cap,
 AS_HELP_STRING([--enable-cap],[enable the search for POSIX capabilities (may require additional headers to be added by hand)]))
 
+# Default off for licensing reasons
 AC_ARG_ENABLE(gdbm,
-AS_HELP_STRING([--disable-gdbm],[turn off search for gdbm library]),
+AS_HELP_STRING([--enable-gdbm],[enable the search for the GDBM library (see the zsh/db/gdbm module)]),
-[gdbm="$enableval"], [gdbm=yes])
+[gdbm="$enableval"], [gdbm=no])
 
 dnl ------------------
 dnl CHECK THE COMPILER
@@ -2541,6 +2542,7 @@ wmemcpy wmemmove wmemset; do
       AC_MSG_NOTICE([all functions found, multibyte support enabled])
       zsh_cv_c_unicode_support=yes
     else
+      # Warns at the end of configure
       AC_MSG_NOTICE([missing functions, multibyte support disabled])
       zsh_cv_c_unicode_support=no
     fi
@@ -3287,13 +3289,18 @@ fi
 echo "See config.modules for installed modules and functions.
 "
 
-case x$LIBS in
+if test x$zsh_cv_c_unicode_support != xyes; then
-  *-lgdbm*)
+  if test "x$zfuncs_absent" = x; then
-  echo "WARNING: zsh will be linked against libgdbm.
+    # The user opted out.
-This means the binary is covered by the GNU General Public License.
+    AC_MSG_WARN([You have chosen to build without multibyte support.])
-This does not affect the source code.
+    AC_MSG_WARN([This configuration may not be suitable for production use. It is known to cause errors in 'make test'. We strongly recommend to re-run configure with --enable-multibyte.])
-Run configure with --disable-gdbm if required."
+  else
-  ;;
+    # Some requisite functions are missing.
-esac
+    AC_MSG_WARN([Multibyte support cannot be enabled: some standard library functions are missing: $zfuncs_absent])
+    AC_MSG_WARN([This configuration may not be suitable for production use. It is known to cause errors in 'make test'. If your system provides those functions, we recommend to re-run configure appropriately.])
+    # If your system doesn't have those functions, consider patching the
+    # test suite and sending the patch to zsh-workers@ for inclusion.
+  fi
+fi
 
 exit 0