48560: add TYPESET_TO_UNSET option to remove initialization of parameters

Changes typeset such that ${newparam-notset} yields "notset" and
"typeset -p newparam" does not show an assignment to the parameter.  This
is similar to the default behavior of bash and ksh, with minor differences
in typeset output.

Also add tests for some POSIX incompatibilities plus minor changes for test
harness robustness.

ChangeLog

@@ -1,3 +1,18 @@
+2021-04-18  Bart Schaefer  <schaefer@zsh.org>
+
+	* 48560: Completion/compinit, Doc/Zsh/builtins.yo,
+	Doc/Zsh/options.yo, Doc/Zsh/params.yo, Src/builtin.c,
+	Src/options.c, Src/params.c, Src/subst.c, Src/zsh.h,
+	Test/D06subscript.ztst, Test/E01options.ztst, Test/E03posix.ztst,
+	Test/V10private.ztst, Test/runtests.zsh, Test/ztst.zsh: add
+	TYPESET_TO_UNSET option, which removes initialization of newly
+	declared parameters such that ${newparam-notset} yields "notset"
+	and "typeset -p newparam" does not show an assignment to the
+	parameter.  This is similar to the default behavior of bash and
+	ksh, with minor differences in typeset output.  Also add tests for
+	some POSIX incompatibilities plus minor changes for test harness
+	robustness.
+
 2021-04-18  Jun-ichi Takimoto  <takimoto-j@kba.biglobe.ne.jp>
 
 	* unposted: Etc/BUGS: remove a bug fixed by 47301
@@ -2909,7 +2924,7 @@ g
 
 2019-02-14  Peter Stephenson  <p.stephenson@samsung.com>
 
-	* see 44062: back off change to ZLE per-line initiialisation,
+	* see 44062: back off change to ZLE per-line initialisation,
 	causing problems after failed reads and apparently not needed
 	for the intended fix of interrupt handling (40305 / 34656ec2).
 
@@ -15632,7 +15647,7 @@ g
 
 	* 32338: Doc/Makefile.in: create Doc/help.txt as an empty file
 	when Util/helpfiles fails, so that the rest of the build does not
-	yeild a spurious error
+	yield a spurious error
 
 	* 32337: Src/params.c: initialize several special parameters to
 	unset for better compatibility in emulation modes; for the same

Completion/compinit

@@ -165,6 +165,7 @@ _comp_options=(
     NO_posixidentifiers
     NO_shwordsplit
     NO_shglob
+    NO_typesettounset
     NO_warnnestedvar
     NO_warncreateglobal
 )

Doc/Zsh/builtins.yo

@@ -1872,7 +1872,11 @@ ifnzman(noderef(Local Parameters))\
 retain their special attributes when made local.
 
 For each var(name)tt(=)var(value) assignment, the parameter
-var(name) is set to var(value).
+var(name) is set to var(value).  If the assignment is omitted and var(name)
+does em(not) refer to an existing parameter, a new parameter is intialized
+to empty string, zero, or empty array (as appropriate), em(unless) the
+shell option tt(TYPESET_TO_UNSET) is set.  When that option is set,
+the parameter attributes are recorded but the parameter remains unset.
 
 If the shell option tt(TYPESET_SILENT) is not set, for each remaining
 var(name) that refers to a parameter that is already set, the name and

Doc/Zsh/options.yo

@@ -1942,6 +1942,16 @@ If the option is set, they will only be shown when parameters are selected
 with the `tt(-m)' option.  The option `tt(-p)' is available whether or not
 the option is set.
 )
+pindex(TYPESET_TO_UNSET)
+pindex(NO_TYPESET_TO_UNSET)
+pindex(TYPESETTOUNSET)
+pindex(NOTYPESETTOUNSET)
+item(tt(TYPESET_TO_UNSET) <K> <S>)(
+When declaring a new parameter with any of the `tt(typeset)' family of
+related commands, the parameter remains unset unless and until a
+value is explicity assigned to it, either in the `tt(typeset)' command
+itself or as a later assignment statement.
+)
 pindex(VERBOSE)
 pindex(NO_VERBOSE)
 pindex(NOVERBOSE)

Doc/Zsh/params.yo

@@ -393,6 +393,11 @@ is compared to the pattern, and the first matching key found is the
 result.  On failure substitutes the length of the array plus one, as
 discussed under the description of `tt(r)', or the empty string for an
 associative array.
+
+Note: Although `tt(i)' may be applied to a scalar substitution to find
+the offset of a substring, the results are likely to be misleading when
+searching within substitutions that yield an empty string, or when
+searching for the empty substring.
 )
 item(tt(I))(
 Like `tt(i)', but gives the index of the last match, or all possible

Src/builtin.c

@@ -2491,6 +2491,8 @@ typeset_single(char *cname, char *pname, Param pm, UNUSED(int func),
 		return NULL;
 	    }
 	}
+	if (isset(TYPESETTOUNSET))
+	    pm->node.flags |= PM_DEFAULTED;
     } else {
 	if (idigit(*pname))
 	    zerrnam(cname, "not an identifier: %s", pname);
@@ -2836,7 +2838,7 @@ bin_typeset(char *name, char **argv, LinkList assigns, Options ops, int func)
 	    unqueue_signals();
 	    return 1;
 	} else if (pm) {
-	    if (!(pm->node.flags & PM_UNSET)
+	    if ((!(pm->node.flags & PM_UNSET) || pm->node.flags & PM_DECLARED)
 		&& (locallevel == pm->level || !(on & PM_LOCAL))) {
 		if (pm->node.flags & PM_TIED) {
 		    if (PM_TYPE(pm->node.flags) != PM_SCALAR) {
@@ -2889,6 +2891,8 @@ bin_typeset(char *name, char **argv, LinkList assigns, Options ops, int func)
 	 *
 	 * Don't attempt to set it yet, it's too early
 	 * to be exported properly.
+	 *
+	 * This may create the array with PM_DEFAULTED.
 	 */
 	asg2.name = asg->name;
 	asg2.flags = 0;
@@ -2930,8 +2934,12 @@ bin_typeset(char *name, char **argv, LinkList assigns, Options ops, int func)
 	if (asg->value.array) {
 	    int flags = (asg->flags & ASG_KEY_VALUE) ? ASSPM_KEY_VALUE : 0;
 	    assignaparam(asg->name, zlinklist2array(asg->value.array, 1), flags);
-	} else if (oldval)
-	    assignsparam(asg0.name, oldval, 0);
+	} else if (asg0.value.scalar || oldval) {
+	    /* We have to undo what we did wrong with asg2 */
+	    apm->node.flags &= ~PM_DEFAULTED;
+	    if (oldval)
+		assignsparam(asg0.name, oldval, 0);
+	}
 	unqueue_signals();
 
 	return 0;

Src/options.c

@@ -259,6 +259,7 @@ static struct optname optns[] = {
 {{NULL, "transientrprompt",   0},			 TRANSIENTRPROMPT},
 {{NULL, "trapsasync",	      0},			 TRAPSASYNC},
 {{NULL, "typesetsilent",      OPT_EMULATE|OPT_BOURNE},	 TYPESETSILENT},
+{{NULL, "typesettounset",     OPT_EMULATE|OPT_BOURNE},	 TYPESETTOUNSET},
 {{NULL, "unset",	      OPT_EMULATE|OPT_BSHELL},	 UNSET},
 {{NULL, "verbose",	      0},			 VERBOSE},
 {{NULL, "vi",		      0},			 VIMODE},

Src/params.c

@@ -2093,7 +2093,8 @@ fetchvalue(Value v, char **pptr, int bracks, int flags)
 	if (sav)
 	    *s = sav;
 	*pptr = s;
-	if (!pm || (pm->node.flags & PM_UNSET))
+	if (!pm || ((pm->node.flags & PM_UNSET) &&
+		    !(pm->node.flags & PM_DECLARED)))
 	    return NULL;
 	if (v)
 	    memset(v, 0, sizeof(*v));
@@ -3055,6 +3056,7 @@ assignsparam(char *s, char *val, int flags)
 	     * Don't warn about anything.
 	     */
 	    flags &= ~ASSPM_WARN;
+	    v->pm->node.flags &= ~PM_DEFAULTED;
 	}
 	*ss = '[';
 	v = NULL;
@@ -3080,6 +3082,7 @@ assignsparam(char *s, char *val, int flags)
     }
     if (flags & ASSPM_WARN)
 	check_warn_pm(v->pm, "scalar", created, 1);
+    v->pm->node.flags &= ~PM_DEFAULTED;
     if (flags & ASSPM_AUGMENT) {
 	if (v->start == 0 && v->end == -1) {
 	    switch (PM_TYPE(v->pm->node.flags)) {
@@ -3232,6 +3235,7 @@ assignaparam(char *s, char **val, int flags)
 
     if (flags & ASSPM_WARN)
 	check_warn_pm(v->pm, "array", created, may_warn_about_nested_vars);
+    v->pm->node.flags &= ~PM_DEFAULTED;
 
     /*
      * At this point, we may have array entries consisting of
@@ -3444,6 +3448,7 @@ sethparam(char *s, char **val)
 	    return NULL;
 	}
     check_warn_pm(v->pm, "associative array", checkcreate, 1);
+    v->pm->node.flags &= ~PM_DEFAULTED;
     setarrvalue(v, val);
     unqueue_signals();
     return v->pm;
@@ -3515,6 +3520,7 @@ assignnparam(char *s, mnumber val, int flags)
 	if (flags & ASSPM_WARN)
 	    check_warn_pm(v->pm, "numeric", 0, 1);
     }
+    v->pm->node.flags &= ~PM_DEFAULTED;
     setnumvalue(v, val);
     unqueue_signals();
     return v->pm;
@@ -3619,6 +3625,7 @@ unsetparam_pm(Param pm, int altflag, int exp)
     else
 	altremove = NULL;
 
+    pm->node.flags &= ~PM_DECLARED;	/* like ksh, not like bash */
     if (!(pm->node.flags & PM_UNSET))
 	pm->gsu.s->unsetfn(pm, exp);
     if (pm->env)
@@ -3652,6 +3659,8 @@ unsetparam_pm(Param pm, int altflag, int exp)
 	}
 
 	zsfree(altremove);
+	if (!(pm->node.flags & PM_SPECIAL))
+	    pm->gsu.s = &stdscalar_gsu;
     }
 
     /*
@@ -4116,6 +4125,11 @@ tiedarrsetfn(Param pm, char *x)
 
     if (*dptr->arrptr)
 	freearray(*dptr->arrptr);
+    else if (pm->ename) {
+	Param altpm = (Param) paramtab->getnode(paramtab, pm->ename);
+	if (altpm)
+	    altpm->node.flags &= ~PM_DEFAULTED;
+    }
     if (x) {
 	char sepbuf[3];
 	if (imeta(dptr->joinchar))
@@ -5035,6 +5049,7 @@ arrfixenv(char *s, char **t)
 
     if (isset(ALLEXPORT))
 	pm->node.flags |= PM_EXPORTED;
+    pm->node.flags &= ~PM_DEFAULTED;
 
     /*
      * Do not "fix" parameters that were not exported
@@ -5839,8 +5854,9 @@ printparamnode(HashNode hn, int printflags)
     Param peer = NULL;
 
     if (p->node.flags & PM_UNSET) {
-	if (printflags & (PRINT_POSIX_READONLY|PRINT_POSIX_EXPORT) &&
-	    p->node.flags & (PM_READONLY|PM_EXPORTED)) {
+	if ((printflags & (PRINT_POSIX_READONLY|PRINT_POSIX_EXPORT) &&
+	     p->node.flags & (PM_READONLY|PM_EXPORTED)) ||
+	    (p->node.flags & PM_DEFAULTED) == PM_DEFAULTED) {
 	    /*
 	     * Special POSIX rules: show the parameter as readonly/exported
 	     * even though it's unset, but with no value.

Src/subst.c

@@ -2563,7 +2563,8 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int pf_flags,
 	     * Handle the (t) flag: value now becomes the type
 	     * information for the parameter.
 	     */
-	    if (v && v->pm && !(v->pm->node.flags & PM_UNSET)) {
+	    if (v && v->pm && ((v->pm->node.flags & PM_DECLARED) ||
+			       !(v->pm->node.flags & PM_UNSET))) {
 		int f = v->pm->node.flags;
 
 		switch (PM_TYPE(f)) {

Src/zsh.h

@@ -1929,8 +1929,10 @@ struct tieddata {
 				   made read-only by the user               */
 #define PM_READONLY_SPECIAL (PM_SPECIAL|PM_READONLY|PM_RO_BY_DESIGN)
 #define PM_DONTIMPORT	(1<<22)	/* do not import this variable              */
+#define PM_DECLARED	(1<<22) /* explicitly named with typeset            */
 #define PM_RESTRICTED	(1<<23) /* cannot be changed in restricted mode     */
 #define PM_UNSET	(1<<24)	/* has null value                           */
+#define PM_DEFAULTED	(PM_DECLARED|PM_UNSET)
 #define PM_REMOVABLE	(1<<25)	/* special can be removed from paramtab     */
 #define PM_AUTOLOAD	(1<<26) /* autoloaded from module                   */
 #define PM_NORESTORE	(1<<27)	/* do not restore value of local special    */
@@ -2536,6 +2538,7 @@ enum {
     TRANSIENTRPROMPT,
     TRAPSASYNC,
     TYPESETSILENT,
+    TYPESETTOUNSET,
     UNSET,
     VERBOSE,
     VIMODE,

Test/D06subscript.ztst

@@ -289,3 +289,8 @@ F:Regression test for workers/42297
 >14 24
 >b b
 >b?rbaz foob?r
+
+  i=1,3
+  [[ ${a[$i]} = ${a[i]} ]]
+0f:Math evaluation of commas in array subscripts
+F:In math, (($i)) should be the same as ((i)), see workers/47748.

Test/E01options.ztst

@@ -1451,3 +1451,18 @@ F:If this test fails at the first unsetopt, refer to P01privileged.ztst.
 0q:RM_STAR_SILENT
 *>zsh: sure you want to delete all 15 files in ${PWD:h}/options.tmp \[yn\]\? ${BEL}(|n)
 *>zsh: sure you want to delete (all <->|more than <->) files in / \[yn\]\? ${BEL}(|n)
+
+  () {
+    local var
+    print ${(t)var}
+  }
+0:(t) returns correct type
+>scalar-local
+
+  () {
+    readonly var
+    typeset -p var
+  }
+0:readonly with typeset -p
+F:compare E03posix.ztst
+>typeset -r var=''

Test/V10private.ztst

@@ -19,14 +19,14 @@
  () {
   print $scalar_test
   private scalar_test
-  print $+scalar_test
+  typeset +m scalar_test
   unset scalar_test
   print $+scalar_test
  }
  print $scalar_test
 0:basic scope hiding
 >toplevel
->1
+>local scalar_test
 >0
 >toplevel
 
@@ -45,14 +45,14 @@
  print $+unset_test
  () {
   private unset_test
-  print $+unset_test
+  typeset +m unset_test
   unset_test=setme
   print $unset_test
  }
  print $+unset_test
 0:variable defined only in scope
 >0
->1
+>local unset_test
 >setme
 >0
 
@@ -62,13 +62,13 @@
   local -Pa array_test=(in function)
   () {
    private array_test
-   print $+array_test
+   typeset +m array_test
   }
   print $array_test
  }
  print $array_test
 0:nested scope with different type, correctly restored
->1
+>local array_test
 >in function
 >top level
 

Test/runtests.zsh

@@ -7,7 +7,7 @@ emulate zsh
 # protect from catastrophic failure of an individual test.
 # We could probably do that with subshells instead.
 
-integer success failure skipped retval
+integer success=0 failure=0 skipped=0 retval
 for file in "${(f)ZTST_testlist}"; do
   $ZTST_exe +Z -f $ZTST_srcdir/ztst.zsh $file
   retval=$?

Test/ztst.zsh

@@ -60,7 +60,7 @@ ZTST_mainopts=(${(kv)options})
 ZTST_testdir=$PWD
 ZTST_testname=$1
 
-integer ZTST_testfailed
+integer ZTST_testfailed=0
 
 # This is POSIX nonsense.  Because of the vague feeling someone, somewhere
 # may one day need to examine the arguments of "tail" using a standard