34322: bug with interface to parsestr() etc.

Was showing up in places like ${(e)...} where command substitution
could reallocate the token string, but actually there was never any
guarantee that the lexer wouldn't do that, so this was always
a bit iffy.

ChangeLog

@@ -1,5 +1,11 @@
 2015-01-18  Peter Stephenson  <p.w.stephenson@ntlworld.com>
 
+	* 34322: Src/Zle/compctl.c, Src/Zle/compresult.c, Src/exec.c,
+	Src/init.c, Src/lex.c, Src/params.c, Src/prompt.c, Src/subst.c,
+	Src/utils.c, Test/D04parameter.ztst: update interface to
+	parsestr()/parsestrnoerr() to ensure correct token string
+	is passed back.
+
 	* 34320: Src/hist.c, Src/lex.c: alias expansion in history of
 	command substitution.
 

Src/Zle/compctl.c

@@ -3853,7 +3853,7 @@ makecomplistflags(Compctl cc, char *s, int incmd, int compadd)
 	    yaptr = get_user_var(uv);
 	if ((tt = cc->explain)) {
 	    tt = dupstring(tt);
-	    if ((cc->mask & CC_EXPANDEXPL) && !parsestr(tt)) {
+	    if ((cc->mask & CC_EXPANDEXPL) && !parsestr(&tt)) {
 		singsub(&tt);
 		untokenize(tt);
 	    }
@@ -3873,7 +3873,7 @@ makecomplistflags(Compctl cc, char *s, int incmd, int compadd)
 	}
     } else if ((tt = cc->explain)) {
 	tt = dupstring(tt);
-	if ((cc->mask & CC_EXPANDEXPL) && !parsestr(tt)) {
+	if ((cc->mask & CC_EXPANDEXPL) && !parsestr(&tt)) {
 	    singsub(&tt);
 	    untokenize(tt);
 	}

Src/Zle/compresult.c

@@ -1090,7 +1090,8 @@ do_single(Cmatch m)
 		    }
 		    if (tryit) {
 			noerrs = 1;
-			parsestr(p);
+			p = dupstring(p);
+			parsestr(&p);
 			singsub(&p);
 			errflag &= ~ERRFLAG_ERROR;
 			noerrs = ne;

Src/exec.c

@@ -3772,19 +3772,20 @@ gethere(char **strp, int typ)
 	*bptr++ = '\n';
     }
     *t = '\0';
+    s = buf;
+    buf = dupstring(buf);
+    zfree(s, bsiz);
     if (!qt) {
 	int ef = errflag;
 
-	parsestr(buf);
+	parsestr(&buf);
 
 	if (!errflag) {
 	    /* Retain any user interrupt error */
 	    errflag = ef | (errflag & ERRFLAG_INT);
 	}
     }
-    s = dupstring(buf);
-    zfree(buf, bsiz);
-    return s;
+    return buf;
 }
 
 /* open here string fd */

Src/init.c

@@ -1197,10 +1197,13 @@ run_init_scripts(void)
 	    if (islogin)
 		sourcehome(".profile");
 	    noerrs = 2;
-	    if (s && !parsestr(s)) {
-		singsub(&s);
-		noerrs = 0;
-		source(s);
+	    if (s) {
+		s = dupstring(s);
+		if (!parsestr(&s)) {
+		    singsub(&s);
+		    noerrs = 0;
+		    source(s);
+		}
 	    }
 	    noerrs = 0;
 	} else

Src/lex.c

@@ -1503,17 +1503,27 @@ dquote_parse(char endchar, int sub)
     return err;
 }
 
-/* Tokenize a string given in s. Parsing is done as in double *
- * quotes.  This is usually called before singsub().          */
+/*
+ * Tokenize a string given in s. Parsing is done as in double
+ * quotes.  This is usually called before singsub().
+ *
+ * parsestr() is noisier, reporting an error if the parse failed.
+ *
+ * On entry, *s must point to a string allocated from the stack of
+ * exactly the right length, i.e. strlen(*s) + 1, as the string
+ * is used as the lexical token string whose memory management
+ * demands this.  Usually the input string will therefore be
+ * the result of an immediately preceding dupstring().
+ */
 
 /**/
 mod_export int
-parsestr(char *s)
+parsestr(char **s)
 {
     int err;
 
     if ((err = parsestrnoerr(s))) {
-	untokenize(s);
+	untokenize(*s);
 	if (err > 32 && err < 127)
 	    zerr("parse error near `%c'", err);
 	else
@@ -1524,18 +1534,20 @@ parsestr(char *s)
 
 /**/
 mod_export int
-parsestrnoerr(char *s)
+parsestrnoerr(char **s)
 {
-    int l = strlen(s), err;
+    int l = strlen(*s), err;
 
     zcontext_save();
-    untokenize(s);
-    inpush(dupstring(s), 0, NULL);
+    untokenize(*s);
+    inpush(dupstring(*s), 0, NULL);
     strinbeg(0);
     lexbuf.len = 0;
-    lexbuf.ptr = tokstr = s;
+    lexbuf.ptr = tokstr = *s;
     lexbuf.siz = l + 1;
     err = dquote_parse('\0', 1);
+    if (tokstr)
+	*s = tokstr;
     *lexbuf.ptr = '\0';
     strinend();
     inpop();

Src/params.c

@@ -1260,7 +1260,8 @@ getarg(char **str, int *inv, Value v, int a2, zlong *w,
     if (ishash && (keymatch || !rev))
 	remnulargs(s);
     if (needtok) {
-	if (parsestr(s))
+	s = dupstring(s);
+	if (parsestr(&s))
 	    return 0;
 	singsub(&s);
     } else if (rev)

Src/prompt.c

@@ -183,7 +183,7 @@ promptexpand(char *s, int ns, char *rs, char *Rs, unsigned int *txtchangep)
 	int oldval = lastval;
 
 	s = dupstring(s);
-	if (!parsestr(s))
+	if (!parsestr(&s))
 	    singsub(&s);
 	/*
 	 * We don't need the special Nularg hack here and we're

Src/subst.c

@@ -1306,7 +1306,7 @@ get_intarg(char **s, int *delmatchp)
     p = dupstring(*s + arglen);
     *s = t + arglen;
     *t = sav;
-    if (parsestr(p))
+    if (parsestr(&p))
 	return -1;
     singsub(&p);
     if (errflag)
@@ -1329,7 +1329,8 @@ subst_parse_str(char **sp, int single, int err)
 
     *sp = s = dupstring(*sp);
 
-    if (!(err ? parsestr(s) : parsestrnoerr(s))) {
+    if (!(err ? parsestr(&s) : parsestrnoerr(&s))) {
+	*sp = s;
 	if (!single) {
             int qt = 0;
 
@@ -1439,7 +1440,8 @@ check_colon_subscript(char *str, char **endp)
     }
     sav = **endp;
     **endp = '\0';
-    if (parsestr(str = dupstring(str)))
+    str = dupstring(str);
+    if (parsestr(&str))
 	return NULL;
     singsub(&str);
     remnulargs(str);

Src/utils.c

@@ -1515,7 +1515,7 @@ checkmailpath(char **s)
 		    setunderscore(*s);
 
 		    u = dupstring(u);
-		    if (! parsestr(u)) {
+		    if (!parsestr(&u)) {
 			singsub(&u);
 			zputs(u, shout);
 			fputc('\n', shout);

Test/D04parameter.ztst

@@ -1663,3 +1663,10 @@
 0:SHLVL appears sensible when about to exit shell
 >2
 >2
+
+# The following tests the return behaviour of parsestr/parsestrnoerr
+  alias param-test-alias='print $'\''\x45xpanded in substitution'\' 
+  param='$(param-test-alias)'
+  print ${(e)param}
+0:Alias expansion in command substitution in parameter evaluation
+>Expanded in substitution