28308/28310: HIST_LEX_WORDS, check for quick history read

ChangeLog

@@ -1,5 +1,9 @@
 2010-10-02  Peter Stephenson  <p.w.stephenson@ntlworld.com>
 
+	* 28310 with 28308 (Bart): Doc/Zsh/options.yo, Src/hist.c,
+	Src/options.c, Src/zsh.h: HIST_LEX_WORDS option and check
+	for full history file read.
+
 	* 28309: Src/subst.c: infinite loop when padding with extra wide
 	characters.
 
@@ -13689,5 +13693,5 @@
 
 *****************************************************
 * This is used by the shell to define $ZSH_PATCHLEVEL
-* $Revision: 1.5090 $
+* $Revision: 1.5091 $
 *****************************************************

Doc/Zsh/options.yo

@@ -836,6 +836,21 @@ command is entered before it vanishes, allowing you to briefly reuse
 or edit the line.  If you want to make it vanish right away without
 entering another command, type a space and press return.
 )
+pindex(HIST_LEX_WORDS)
+pindex(NO_HIST_LEX_WORDS)
+pindex(HISTLEXWORDS)
+pindex(NOHISTLEXWORDS)
+item(tt(HIST_LEX_WORDS))(
+By default, shell history that is read in from files is split into
+words on all white space.  This means that arguments with quoted
+whitespace are not correctly handled, with the consequence that
+references to words in history lines that have been read from a file
+may be inaccurate.  When this option is set, words read in from a
+history file are divided up in a similar fashion to normal shell
+command line handling.  Although this produces more accurately delimited
+words, if the size of the history file is large this can be slow.  Trial
+and error is necessary to decide.
+)
 pindex(HIST_NO_FUNCTIONS)
 pindex(NO_HIST_NO_FUNCTIONS)
 pindex(HISTNOFUNCTIONS)

Src/hist.c

@@ -2232,7 +2232,6 @@ readhistfile(char *fn, int err, int readflags)
     struct stat sb;
     int nwordpos, nwords, bufsiz;
     int searching, newflags, l, ret;
-    LinkList wordlist;
    
     if (!fn && !(fn = getsparam("HISTFILE")))
 	return;
@@ -2336,60 +2335,81 @@ readhistfile(char *fn, int err, int readflags)
 		he->ftim = ftim;
 
 	    /*
-	     * Divide up the words.  Attempt to do this using the lexer.
+	     * Divide up the words.
 	     */
 	    nwordpos = 0;
 	    start = pt;
-	    wordlist = bufferwords(NULL, pt, NULL);
-	    he->nwords = countlinknodes(wordlist);
-	    if (2*he->nwords > nwords) {
-		nwords = 2*he->nwords;
-		words = (short *)realloc(words, nwords*sizeof(short));
-	    }
-	    while (firstnode(wordlist)) {
-		char *word = uremnode(wordlist, firstnode(wordlist));
-		
-		while (inblank(*pt))
-		    pt++;
-		if (!strpfx(word, pt)) {
-		    int bad = 0;
-		    /*
-		     * Oddity 1: newlines turn into semicolons.
-		     */
-		    if (!strcmp(word, ";"))
-			continue;
-		    /*
-		     * Oddity 2: !'s turn into |'s.
-		     */
-		    while (*pt) {
-			if (!*word) {
-			    bad = 1;
-			    break;
-			}
-			if (*pt == *word ||
-			    (*pt == '!' && *word == '|')) {
-			    pt++;
-			    word++;
-			} else {
-			    bad = 1;
-			    break;
-			}
-		    }
-		    if (bad) {
-#ifdef DEBUG
-			dputs(ERRMSG("bad wordsplit reading history: %s\nat: %s"
-				     "\nword: %s"),
-			      start, pt, word);
-#endif
-			words[nwordpos++] = pt - start;
-			pt += strlen(pt);
-			words[nwordpos++] = pt - start;
-			break;
-		    }
+	    if (isset(HISTLEXWORDS) && !(readflags & HFILE_FAST)) {
+		/*
+		 * Attempt to do this using the lexer.
+		 */
+		LinkList wordlist = bufferwords(NULL, pt, NULL);
+		he->nwords = countlinknodes(wordlist);
+		if (2*he->nwords > nwords) {
+		    nwords = 2*he->nwords;
+		    words = (short *)realloc(words, nwords*sizeof(short));
 		}
-		words[nwordpos++] = pt - start;
-		pt += strlen(word);
-		words[nwordpos++] = pt - start;
+		while (firstnode(wordlist)) {
+		    char *word = uremnode(wordlist, firstnode(wordlist));
+		
+		    while (inblank(*pt))
+			pt++;
+		    if (!strpfx(word, pt)) {
+			int bad = 0;
+			/*
+			 * Oddity 1: newlines turn into semicolons.
+			 */
+			if (!strcmp(word, ";"))
+			    continue;
+			/*
+			 * Oddity 2: !'s turn into |'s.
+			 */
+			while (*pt) {
+			    if (!*word) {
+				bad = 1;
+				break;
+			    }
+			    if (*pt == *word ||
+				(*pt == '!' && *word == '|')) {
+				pt++;
+				word++;
+			    } else {
+				bad = 1;
+				break;
+			    }
+			}
+			if (bad) {
+#ifdef DEBUG
+			    dputs(ERRMSG("bad wordsplit reading history: "
+					 "%s\nat: %s\nword: %s"),
+				  start, pt, word);
+#endif
+			    words[nwordpos++] = pt - start;
+			    pt += strlen(pt);
+			    words[nwordpos++] = pt - start;
+			    break;
+			}
+		    }
+		    words[nwordpos++] = pt - start;
+		    pt += strlen(word);
+		    words[nwordpos++] = pt - start;
+		}
+	    } else {
+		do {
+		    while (inblank(*pt))
+			pt++;
+		    if (*pt) {
+			if (nwordpos >= nwords)
+			    words = (short *)
+				realloc(words, (nwords += 64)*sizeof(short));
+			words[nwordpos++] = pt - start;
+			while (*pt && !inblank(*pt))
+			    pt++;
+			words[nwordpos++] = pt - start;
+		    }
+		} while (*pt);
+
+		he->nwords = nwordpos/2;
 	    }
 
 	    if (he->nwords) {

Src/options.c

@@ -149,6 +149,7 @@ static struct optname optns[] = {
 {{NULL, "histignorealldups",  0},			 HISTIGNOREALLDUPS},
 {{NULL, "histignoredups",     0},			 HISTIGNOREDUPS},
 {{NULL, "histignorespace",    0},			 HISTIGNORESPACE},
+{{NULL, "histlexwords",	      0},			 HISTLEXWORDS},
 {{NULL, "histnofunctions",    0},			 HISTNOFUNCTIONS},
 {{NULL, "histnostore",	      0},			 HISTNOSTORE},
 {{NULL, "histsubstpattern",   OPT_EMULATE},              HISTSUBSTPATTERN},

Src/zsh.h

@@ -1942,6 +1942,7 @@ enum {
     HISTIGNOREALLDUPS,
     HISTIGNOREDUPS,
     HISTIGNORESPACE,
+    HISTLEXWORDS,
     HISTNOFUNCTIONS,
     HISTNOSTORE,
     HISTREDUCEBLANKS,