24853: use metafied strings for inner loops over history

ChangeLog

@@ -1,5 +1,13 @@
 2008-04-20  Peter Stephenson  <p.w.stephenson@ntlworld.com>
- 
+
+	* 24853: Src/utils.c, Src/zsh.h, Src/Zle/complist.c,
+	Src/Zle/zle.h, Src/Zle/zle_hist.c, Src/Zle/zle_main.c,
+	Src/Zle/zle_misc.c, Src/Zle/zle_move.c, Src/Zle/zle_refresh.c,
+	Src/Zle/zle_thingy.c, Src/Zle/zle_utils.c, Src/Zle/zle_vi.c,
+	Src/Zle/zle_word.c:  use metafied strings for inner loops
+	involving history, with consequent changes for similar code
+	such as execute-name-command, plus some utility functions.
+
 	* 24852: Src/zle_main.c (Jun T.), Src/Zle/zle_vi.c: more
 	combining chars fallout.
 

Src/Zle/complist.c

@@ -2432,7 +2432,7 @@ domenuselect(Hookdef dummy, Chdata dat)
         }
         first = 0;
         if (mode == MM_INTER)
-	    statusline = stringaszleline(status, 0, &statusll, NULL, NULL);
+	    statusline = status;
         else if (mode) {
             int l = sprintf(status, "%s%sisearch%s: ",
                             ((msearchstate & MS_FAILED) ? "failed " : ""),
@@ -2441,17 +2441,12 @@ domenuselect(Hookdef dummy, Chdata dat)
 
             strncat(status, msearchstr, MAX_STATUS - l - 1);
 
-            statusline = stringaszleline(status, 0, &statusll, NULL, NULL);
+            statusline = status;
         } else {
             statusline = NULL;
-            statusll = 0;
         }
         zrefresh();
-	if (statusline) {
+	statusline = NULL;
-	    free(statusline);
-	    statusline = NULL;
-	    statusll = 0;
-	}
         inselect = 1;
         if (noselect) {
             broken = 1;
@@ -2622,12 +2617,10 @@ domenuselect(Hookdef dummy, Chdata dat)
 	    if (nmatches < 1 || !minfo.cur || !*(minfo.cur)) {
 		nolist = 1;
                 if (mode == MM_INTER) {
-                    statusline = stringaszleline(status, 0,
+                    statusline = status;
-						 &statusll, NULL, NULL);
                 } else {
 		    /* paranoia */
 		    statusline = NULL;
-		    statusll = 0;
 		}
 		if (nmessages) {
 		    showinglist = -2;
@@ -2645,11 +2638,7 @@ domenuselect(Hookdef dummy, Chdata dat)
 		    zrefresh();
 		    showinglist = clearlist = 0;
 		}
-		if (statusline) {
+		statusline = NULL;
-		    free(statusline);
-		    statusline = NULL;
-		    statusll = 0;
-		}
 
 		goto getk;
 	    }
@@ -2763,19 +2752,13 @@ domenuselect(Hookdef dummy, Chdata dat)
 
             if (nolist) {
                 if (mode == MM_INTER) {
-                    statusline = stringaszleline(status, 0,
+                    statusline = status;
-						 &statusll, NULL, NULL);
                 } else {
 		    /* paranoia */
 		    statusline = NULL;
-		    statusll = 0;
 		}
                 zrefresh();
-		if (statusline) {
+		statusline = NULL;
-		    free(statusline);
-		    statusline = NULL;
-		    statusll = 0;
-		}
                 goto getk;
             }
             if (mode)

Src/Zle/zle.h

@@ -74,9 +74,21 @@ typedef wint_t   ZLE_INT_T;
 #define LASTFULLCHAR	lastchar_wide
 #define LASTFULLCHAR_T  ZLE_INT_T
 
-/* We may need to handle combining character alignment */
+/*
+ * We may need to handle combining character alignment.
+ * The following fix up the position of the cursor so that it
+ * never ends up over a zero-width punctuation character following
+ * an alphanumeric character.  The first is used if we were
+ * moving the cursor left, the second if we were moving right or
+ * if something under the cursor may have changed.
+ */
 #define CCLEFT()	alignmultiwordleft(&zlecs, 1)
 #define CCRIGHT()	alignmultiwordright(&zlecs, 1)
+/*
+ * Same for any other position
+ */
+#define CCLEFTPOS(pos)	alignmultiwordleft(&pos, 1)
+#define CCRIGHTPOS(pos)	alignmultiwordright(&pos, 1)
 /*
  * Increment or decrement the cursor position, skipping over
  * combining characters.
@@ -151,6 +163,8 @@ static inline int ZS_strncmp(ZLE_STRING_T s1, ZLE_STRING_T s2, size_t l)
 /* Combining character alignment: none in this mode */
 #define CCLEFT()
 #define CCRIGHT()
+#define CCLEFTPOS()
+#define CCRIGHTPOS()
 /*
  * Increment or decrement the cursor position: simple in this case.
  */
@@ -235,7 +249,13 @@ struct modifier {
 
 #define CUT_FRONT   (1<<0)   /* Text goes in front of cut buffer */
 #define CUT_REPLACE (1<<1)   /* Text replaces cut buffer */
-#define CUT_RAW     (1<<2)   /* Raw character counts (not used in cut itself) */
+#define CUT_RAW     (1<<2)   /*
+			      * Raw character counts (not used in cut itself).
+			      * This is used when the values are offsets
+			      * into the zleline array rather than numbers
+			      * of visible characters directly input by
+			      * the user.
+			      */
 
 /* undo system */
 

Src/Zle/zle_hist.c

@@ -54,8 +54,9 @@ int previous_search_len = 0;
 
 
 struct zle_text {
-    ZLE_STRING_T text;
+    /* Metafied, NULL-terminated string */
-    int len;
+    char *text;
+    /* 1 if we have allocated space for text */
     int alloced;
 };
 
@@ -67,30 +68,18 @@ struct zle_text {
  * Each use of this must have a matching zletextfree() in order
  * to free up the allocated line, if any.  (N.B.: each use *of
  * the function*, not just each use of a struct zle_text.)
- *
- * TODO: This is quite inefficient.  We could convert zlinecmp and
- * zlinefind to take a metafied string as input and acquire a (wide)
- * character from it whenever needed, which would also require storing
- * zle_text as a metafied string in remember_edits().  However, the
- * following is good enough for now (although searching a really huge
- * history might not be so much fun).
  */
 
 static void
 zletext(Histent ent, struct zle_text *zt)
 {
-    char *duptext;
-
     if (ent->zle_text) {
 	zt->text = ent->zle_text;
-	zt->len = ent->zle_len;
 	zt->alloced = 0;
 	return;
     }
 
-    duptext = ztrdup(ent->node.nam);
+    zt->text = ztrdup(ent->node.nam);
-    zt->text = stringaszleline(duptext, 0, &zt->len, NULL, NULL);
-    zsfree(duptext);
     zt->alloced = 1;
 }
 
@@ -111,14 +100,15 @@ remember_edits(void)
 {
     Histent ent = quietgethist(histline);
     if (ent) {
-	if (!ent->zle_text || ent->zle_len != zlell ||
+	char *line = 
-	    ZS_memcmp(ent->zle_text, zleline, zlell) != 0) {
+	    zlemetaline ? zlemetaline :
+	    zlelineasstring(zleline, zlell, 0, NULL, NULL, 0);
+	if (!ent->zle_text || strcmp(line, ent->zle_text) != 0) {
 	    if (ent->zle_text)
 		free(ent->zle_text);
-	    ent->zle_text = zalloc(zlell * ZLE_CHAR_SIZE);
+	    ent->zle_text = zlemetaline ? ztrdup(line) : line;
-	    ent->zle_len = zlell;
+	} else if (!zlemetaline)
-	    ZS_memcpy(ent->zle_text, zleline, zlell);
+	    free(line);
-	}
     }
 }
 
@@ -132,7 +122,6 @@ forget_edits(void)
 	if (he->zle_text) {
 	    free(he->zle_text);
 	    he->zle_text = NULL;
-	    he->zle_len = 0;
 	}
     }
 }
@@ -150,65 +139,99 @@ forget_edits(void)
  */
 
 static int
-zlinecmp(ZLE_STRING_T histp, int histl, ZLE_STRING_T inputp, int inputl)
+zlinecmp(const char *histp, const char *inputp)
 {
-    int cnt;
+    const char *hptr = histp, *iptr = inputp;
+#ifdef MULTIBYTE_SUPPORT
+    mbstate_t hstate, istate;
+#endif
 
-    if (histl < inputl) {
+    while (*hptr == *iptr) {
-	/* Not identical, second string is not a prefix. */
+	hptr++;
-	return 3;
+	iptr++;
     }
 
-    if (!ZS_memcmp(histp, inputp, inputl)) {
+    if (!*iptr) {
-	/* Common prefix is identical */
+	if (!*hptr) {
-	/* If lines are identical return 0 */
+	    /* strings are the same */
-	if (histl == inputl)
 	    return 0;
-	/* Second string is a prefix of the first */
+	} else {
-	return -1;
+	    /* inputp is a prefix */
+	    return -1;
+	}
     }
 
-    for (cnt = inputl; cnt; cnt--) {
+#ifdef MULTIBYTE_SUPPORT
-	if ((ZLE_INT_T)*inputp++ != ZC_tolower(*histp++))
+    memset(&hstate, 0, sizeof(hstate));
+    memset(&istate, 0, sizeof(istate));
+#endif
+
+    /* look for lower case versions */
+    while (*histp && *inputp) {
+#ifdef MULTIBYTE_SUPPORT
+	wint_t hwc, iwc;
+	int hlen, ilen;
+
+	hlen = mb_metacharlenconv_r(histp, &hwc, &hstate);
+	ilen = mb_metacharlenconv_r(inputp, &iwc, &istate);
+
+	if (hwc == WEOF || iwc == WEOF) {
+	    /* can't convert, compare input characters */
+	    if (ilen != hlen || memcmp(histp, inputp, hlen) != 0)
+		return 3;
+	} else if (towlower(hwc) != iwc)
 	    return 3;
+
+	histp += hlen;
+	inputp += ilen;
+#else
+    	if (tulower(*histp++) != *inputp++)
+	    return 3;
+#endif
     }
-    /* Is second string is lowercase version of first? */
+    if (!*inputp) {
-    if (histl == inputl)
+	/* one string finished, if it's the input... */
-	return 1;
+	if (!*histp)
-    /* Second string is lowercase prefix of first */
+	    return 1;		/* ...same, else */
-    return 2;
+	else
+	    return 2;		/* ...prefix */
+    }
+    /* Different */
+    return 3;
 }
 
 
 /*
- * Search for needle in haystack.  Haystack and needle are ZLE strings
+ * Search for needle in haystack.  Haystack and needle are metafied strings.
- * of the indicated length.  Start the search at position
+ * Start the search at position pos in haystack.
- * pos in haystack.  Search forward if dir > 0, otherwise search
+ * Search forward if dir > 0, otherwise search backward.
- * backward.  sens is used to test against the return value of linecmp.
+ * sens is used to test against the return value of linecmp.
+ *
+ * Return the pointer to the location in haystack found, else NULL.
+ *
+ * We assume we'll only find needle at some sensible position in a multibyte
+ * string, so we don't bother calculating multibyte character lengths for
+ * incrementing and decrementing the search position.
  */
 
-static ZLE_STRING_T
+static char *
-zlinefind(ZLE_STRING_T haystack, int haylen, int pos,
+zlinefind(char *haystack, int pos, char *needle, int dir, int sens)
-	  ZLE_STRING_T needle, int needlen, int dir, int sens)
 {
-    ZLE_STRING_T s = haystack + pos;
+    char *s = haystack + pos;
-    int slen = haylen - pos;
 
     if (dir > 0) {
-	while (slen) {
+	while (*s) {
-	    if (zlinecmp(s, slen, needle, needlen) < sens)
+	    if (zlinecmp(s, needle) < sens)
 		return s;
 	    s++;
-	    slen--;
 	}
     } else {
 	for (;;) {
-	    if (zlinecmp(s, slen, needle, needlen) < sens)
+	    if (zlinecmp(s, needle) < sens)
 		return s;
 	    if (s == haystack)
 		break;
 	    s--;
-	    slen++;
 	}
     }
 
@@ -257,7 +280,7 @@ upline(void)
 	if ((zlecs += lastcol) >= x) {
 	    zlecs = x;
 	    if (zlecs > findbol() && invicmdmode())
-		zlecs--;
+		DECCS();
 	}
     }
     return n;
@@ -341,7 +364,7 @@ downline(void)
 	if ((zlecs += lastcol) >= x) {
 	    zlecs = x;
 	    if (zlecs > findbol() && invicmdmode())
-		zlecs--;
+		DECCS();
 	}
     }
     return n;
@@ -421,16 +444,26 @@ downhistory(UNUSED(char **args))
     return 0;
 }
 
-static int histpos, srch_hl, srch_cs = -1;
+/*
-static ZLE_STRING_T srch_str;
+ * Values remembered for history searches to enable repetition.
+ * srch_hl remembers the old value of histline, to see if it's changed
+ *   since the last search.
+ * srch_cs remembers the old value of zlecs for the same purpose (it is
+ *   not use for any other purpose, i.e. does not need to be a valid
+ *   index into anything).
+ * srch_str is the metafied search string, as extracted from the start
+ *   of zleline.
+ */
+static int srch_hl, srch_cs = -1;
+static char *srch_str;
 
 /**/
 int
 historysearchbackward(char **args)
 {
     Histent he;
-    int n = zmult, hp;
+    int n = zmult, histpos;
-    ZLE_STRING_T str;
+    char *str;
     struct zle_text zt;
 
     if (zmult < 0) {
@@ -440,46 +473,45 @@ historysearchbackward(char **args)
 	zmult = n;
 	return ret;
     }
-    if (*args)
+    if (*args) {
-	str = stringaszleline(*args, 0, &hp, NULL, NULL);
+	str = *args;
-    else {
+    } else {
+	char *line = zlelineasstring(zleline, zlell, 0, NULL, NULL, 0);
 	if (histline == curhist || histline != srch_hl || zlecs != srch_cs ||
-	    mark != 0 || ZS_memcmp(srch_str, zleline, histpos) != 0) {
+	    mark != 0 || strncmp(srch_str, line, histpos) != 0) {
-	    zfree(srch_str, histpos);
+	    free(srch_str);
-	    for (histpos = 0; histpos < zlell && !ZC_iblank(zleline[histpos]); histpos++) ;
+	    for (histpos = 0; histpos < zlell && !ZC_iblank(zleline[histpos]);
+		 histpos++)
+		;
 	    if (histpos < zlell)
 		histpos++;
-	    srch_str = zalloc(histpos * ZLE_CHAR_SIZE);
+	    /* ensure we're not on a combining character */
-	    ZS_memcpy(srch_str, zleline, histpos);
+	    CCRIGHTPOS(histpos);
+	    /* histpos from now on on is an index into the metafied string */
+	    srch_str = zlelineasstring(zleline, histpos, 0, NULL, NULL, 0);
 	}
+	free(line);
 	str = srch_str;
-	hp = histpos;
     }
-    if (!(he = quietgethist(histline))) {
+    if (!(he = quietgethist(histline)))
-	if (*args)
-	    free(str);
 	return 1;
-    }
+
     while ((he = movehistent(he, -1, hist_skip_flags))) {
 	if (isset(HISTFINDNODUPS) && he->node.flags & HIST_DUP)
 	    continue;
 	zletext(he, &zt);
-	if (zlinecmp(zt.text, zt.len, str, hp) < 0 &&
+	if (zlinecmp(zt.text, str) < 0 &&
-	    (*args || zlell != zt.len || ZS_memcmp(zt.text, str, zlell))) {
+	    (*args || strcmp(zt.text, str) != 0)) {
 	    if (--n <= 0) {
 		zle_setline(he);
 		srch_hl = histline;
 		srch_cs = zlecs;
-		if (*args)
-		    free(str);
 		zletextfree(&zt);
 		return 0;
 	    }
 	}
 	zletextfree(&zt);
     }
-    if (*args)
-	free(str);
     return 1;
 }
 
@@ -488,8 +520,8 @@ int
 historysearchforward(char **args)
 {
     Histent he;
-    int n = zmult, hp;
+    int n = zmult, histpos;
-    ZLE_STRING_T str;
+    char *str;
     struct zle_text zt;
 
     if (zmult < 0) {
@@ -499,46 +531,43 @@ historysearchforward(char **args)
 	zmult = n;
 	return ret;
     }
-    if (*args)
+    if (*args) {
-	str = stringaszleline(*args, 0, &hp, NULL, NULL);
+	str = *args;
-    else {
+    } else {
+	char *line = zlelineasstring(zleline, zlell, 0, NULL, NULL, 0);
 	if (histline == curhist || histline != srch_hl || zlecs != srch_cs ||
-	    mark != 0 || ZS_memcmp(srch_str, zleline, histpos) != 0) {
+	    mark != 0 || strncmp(srch_str, line, histpos) != 0) {
-	    zfree(srch_str, histpos * ZLE_CHAR_SIZE);
+	    free(srch_str);
-	    for (histpos = 0; histpos < zlell && !ZC_iblank(zleline[histpos]); histpos++) ;
+	    for (histpos = 0; histpos < zlell && !ZC_iblank(zleline[histpos]);
+		 histpos++)
+		;
 	    if (histpos < zlell)
 		histpos++;
-	    srch_str = zalloc(histpos * ZLE_CHAR_SIZE);
+	    CCRIGHT();
-	    ZS_memcpy(srch_str, zleline, histpos);
+	    srch_str = zlelineasstring(zleline, histpos, 0, NULL, NULL, 0);
 	}
+	free(line);
 	str = srch_str;
-	hp = histpos;
     }
-    if (!(he = quietgethist(histline))) {
+    if (!(he = quietgethist(histline)))
-	if (*args)
-	    free(str);
 	return 1;
-    }
+
     while ((he = movehistent(he, 1, hist_skip_flags))) {
 	if (isset(HISTFINDNODUPS) && he->node.flags & HIST_DUP)
 	    continue;
 	zletext(he, &zt);
-	if (zlinecmp(zt.text, zt.len, str, hp) < (he->histnum == curhist) &&
+	if (zlinecmp(zt.text, str) < (he->histnum == curhist) &&
-	    (*args || zlell != zt.len || ZS_memcmp(zt.text, str, zlell))) {
+	    (*args || strcmp(zt.text, str) != 0)) {
 	    if (--n <= 0) {
 		zle_setline(he);
 		srch_hl = histline;
 		srch_cs = zlecs;
-		if (*args)
-		    free(str);
 		zletextfree(&zt);
 		return 0;
 	    }
 	}
 	zletextfree(&zt);
     }
-    if (*args)
-	free(str);
     return 1;
 }
 
@@ -741,29 +770,21 @@ insertlastword(char **args)
 void
 zle_setline(Histent he)
 {
+    int remetafy;
+    if (zlemetaline) {
+	unmetafy_line();
+	remetafy = 1;
+    } else
+	remetafy = 0;
     remember_edits();
     mkundoent();
     histline = he->histnum;
 
-    if (he->zle_text) {
+    setline(he->zle_text ? he->zle_text : he->node.nam, ZSL_COPY|ZSL_TOEND);
-	/*
-	 * Optimise out conversion to metafied string and back.
-	 * Remember convention of extra 2 characters spare.
-	 */
-	free(zleline);
-	linesz = zlell = he->zle_len;
-	zleline = zalloc((zlell + 2) * ZLE_CHAR_SIZE);
-	ZS_memcpy(zleline, he->zle_text, zlell);
-
-	if ((zlecs = zlell) && invicmdmode())
-	    DECCS();
-    } else {
-	setline(he->node.nam, ZSL_COPY|ZSL_TOEND);
-    }
-    /* Move right if we're on a zero-width combining character */
-    CCRIGHT();
     setlastline();
     clearlist = 1;
+    if (remetafy)
+	metafy_line();
 }
 
 /**/
@@ -783,6 +804,8 @@ int
 zle_goto_hist(int ev, int n, int skipdups)
 {
     Histent he = quietgethist(ev);
+    char *line = zlelineasstring(zleline, zlell, 0, NULL, NULL, 1);
+
     if (!he || !(he = movehistent(he, n, hist_skip_flags)))
 	return 1;
     if (skipdups && n) {
@@ -793,7 +816,7 @@ zle_goto_hist(int ev, int n, int skipdups)
 	    int ret;
 
 	    zletext(he, &zt);
-	    ret = zlinecmp(zt.text, zt.len, zleline, zlell);
+	    ret = zlinecmp(zt.text, line);
 	    zletextfree(&zt);
 	    if (ret)
 		break;
@@ -961,7 +984,7 @@ get_isrch_spot(int num, int *hlp, int *posp, int *csp, int *lenp, int *dirp, int
     *nomatch = (isrch_spots[num].flags & ISS_FAILING);
 }
 
-#define ISEARCH_PROMPT		ZWS("failing XXX-i-search: ")
+#define ISEARCH_PROMPT		"failing XXX-i-search: "
 #define NORM_PROMPT_POS		8
 #define FIRST_SEARCH_CHAR	(NORM_PROMPT_POS + 14)
 
@@ -969,17 +992,18 @@ get_isrch_spot(int num, int *hlp, int *posp, int *csp, int *lenp, int *dirp, int
 static void
 doisearch(char **args, int dir)
 {
-    ZLE_STRING_T ibuf = zhalloc(80 * ZLE_CHAR_SIZE);
+    char *ibuf = zhalloc(80);
-    ZLE_STRING_T sbuf = ibuf + FIRST_SEARCH_CHAR;
+    char *sbuf = ibuf + FIRST_SEARCH_CHAR;
-    ZLE_STRING_T last_line = NULL;
+    char *last_line = NULL;
     struct zle_text zt;
     int sbptr = 0, top_spot = 0, pos, sibuf = 80;
     int nomatch = 0, skip_line = 0, skip_pos = 0;
     int odir = dir, sens = zmult == 1 ? 3 : 1;
-    int hl = histline, savekeys = -1, feep = 0, last_len;
+    int hl = histline, savekeys = -1, feep = 0;
     Thingy cmd;
     char *okeymap;
     Histent he;
+    ZleIntFunc exitfn = (ZleIntFunc)0;
 
     if (!(he = quietgethist(hl)))
 	return;
@@ -994,18 +1018,21 @@ doisearch(char **args, int dir)
 	ungetbytes(arg, len);
     }
 
-    ZS_strcpy(ibuf, ISEARCH_PROMPT);
+    strcpy(ibuf, ISEARCH_PROMPT);
-    ZS_memcpy(ibuf + NORM_PROMPT_POS, (dir == 1) ? ZWS("fwd") : ZWS("bck"), 3);
+    /* careful with fwd/bck: we don't want the NULL copied */
-    remember_edits();
+    memcpy(ibuf + NORM_PROMPT_POS, (dir == 1) ? "fwd" : "bck", 3);
     okeymap = ztrdup(curkeymapname);
-    zletext(he, &zt);
     selectkeymap("main", 1);
-    pos = zlecs;
+
+    metafy_line();
+    remember_edits();
+    zletext(he, &zt);
+    pos = zlemetacs;
     for (;;) {
 	/* Remember the current values in case search fails (doesn't push). */
-	set_isrch_spot(top_spot, hl, pos, zlecs, sbptr, dir, nomatch);
+	set_isrch_spot(top_spot, hl, pos, zlemetacs, sbptr, dir, nomatch);
-	if (sbptr == 1 && sbuf[0] == ZWC('^')) {
+	if (sbptr == 1 && sbuf[0] == '^') {
-	    zlecs = 0;
+	    zlemetacs = 0;
     	    nomatch = 0;
 	    statusline = ibuf + NORM_PROMPT_POS;
 	} else if (sbptr > 0) {
@@ -1016,21 +1043,28 @@ doisearch(char **args, int dir)
 	     */
 	    if (last_line)
 		free(last_line);
-	    last_line = zalloc(zt.len * ZLE_CHAR_SIZE);
+	    last_line = ztrdup(zt.text);
-	    ZS_memcpy(last_line, zt.text, zt.len);
-	    last_len = zt.len;
 
 	    for (;;) {
-		ZLE_STRING_T t;
+		char *t;
 
+		/*
+		 * If instructed, move past a match position:
+		 * backwards if searching backwards (skipping
+		 * the line if we're at the start), forwards
+		 * if searching forwards (skipping a line if we're
+		 * at the end).
+		 */
 		if (skip_pos) {
 		    if (dir < 0) {
 			if (pos == 0)
 			    skip_line = 1;
 			else
-			    pos -= 1;
+			    pos = backwardmetafiedchar(zlemetaline,
-		    } else if (sbuf[0] != ZWC('^')) {
+						       zlemetaline + pos,
-			if (pos >= zt.len - 1)
+						       NULL) - zlemetaline;
+		    } else if (sbuf[0] != '^') {
+			if (pos >= strlen(zt.text) - 1)
 			    skip_line = 1;
 			else
 			    pos += 1;
@@ -1038,25 +1072,33 @@ doisearch(char **args, int dir)
 			skip_line = 1;
 		    skip_pos = 0;
 		}
-		if (!skip_line && ((sbuf[0] == ZWC('^')) ?
+		/*
-		    (t = zlinecmp(zt.text, zt.len, sbuf + 1, sbptr - 1) < sens
+		 * First search for a(nother) match within the
-		     ? zt.text : NULL) :
+		 * current line, unless we've been told to skip it.
-		    (t = zlinefind(zt.text, zt.len, pos, sbuf,
+		 */
-				   sbptr, dir, sens)))) {
+		sbuf[sbptr] = '\0';
+		if (!skip_line && ((sbuf[0] == '^') ?
+				   (t = (zlinecmp(zt.text, sbuf + 1) < sens
+					 ? zt.text : NULL)) :
+		    (t = zlinefind(zt.text, pos, sbuf, dir, sens)))) {
 		    zle_setline(he);
 		    pos = t - zt.text;
-		    zlecs = pos +
+		    zlemetacs = pos +
-			(dir == 1 ? sbptr - (sbuf[0] == ZWC('^')) : 0);
+			(dir == 1 ? sbptr - (sbuf[0] == '^') : 0);
 	    	    nomatch = 0;
 		    statusline = ibuf + NORM_PROMPT_POS;
 		    break;
 		}
+		/*
+		 * If not found within that line, move through
+		 * the history to try again.
+		 */
 		if (!(zlereadflags & ZLRF_HISTORY)
 		 || !(he = movehistent(he, dir, hist_skip_flags))) {
 		    if (sbptr == (int)isrch_spots[top_spot-1].len
 		     && (isrch_spots[top_spot-1].flags & ISS_FAILING))
 			top_spot--;
-		    get_isrch_spot(top_spot, &hl, &pos, &zlecs, &sbptr,
+		    get_isrch_spot(top_spot, &hl, &pos, &zlemetacs, &sbptr,
 				   &dir, &nomatch);
 		    if (!nomatch) {
 			feep = 1;
@@ -1072,18 +1114,18 @@ doisearch(char **args, int dir)
 		hl = he->histnum;
 		zletextfree(&zt);
 		zletext(he, &zt);
-		pos = (dir == 1) ? 0 : zt.len;
+		pos = (dir == 1) ? 0 : strlen(zt.text);
-		skip_line = isset(HISTFINDNODUPS) ? !!(he->node.flags & HIST_DUP)
+		skip_line = isset(HISTFINDNODUPS)
-		    : (zt.len == last_len &&
+		    ? !!(he->node.flags & HIST_DUP)
-		       !ZS_memcmp(zt.text, last_line, zt.len));
+		    : !strcmp(zt.text, last_line);
 	    }
 	} else {
 	    top_spot = 0;
     	    nomatch = 0;
 	    statusline = ibuf + NORM_PROMPT_POS;
 	}
-	sbuf[sbptr] = ZWC('_');
+	sbuf[sbptr] = '_';
-	statusll = sbuf - statusline + sbptr + 1;
+	sbuf[sbptr+1] = '\0';
     ref:
 	zrefresh();
 	if (!(cmd = getkeycmd()) || cmd == Th(z_sendbreak)) {
@@ -1093,7 +1135,7 @@ doisearch(char **args, int dir)
 	    zle_setline(he);
 	    zletextfree(&zt);
 	    zletext(he, &zt);
-	    zlecs = i;
+	    zlemetacs = i;
 	    break;
 	}
 	if(cmd == Th(z_clearscreen)) {
@@ -1109,7 +1151,7 @@ doisearch(char **args, int dir)
 	} else if(cmd == Th(z_vibackwarddeletechar) ||
 	    	cmd == Th(z_backwarddeletechar)) {
 	    if (top_spot)
-		get_isrch_spot(--top_spot, &hl, &pos, &zlecs, &sbptr,
+		get_isrch_spot(--top_spot, &hl, &pos, &zlemetacs, &sbptr,
 			       &dir, &nomatch);
 	    else
 		feep = 1;
@@ -1120,67 +1162,66 @@ doisearch(char **args, int dir)
 	    he = quietgethist(hl);
 	    zletextfree(&zt);
 	    zletext(he, &zt);
-	    if (nomatch || !sbptr || (sbptr == 1 && sbuf[0] == ZWC('^'))) {
+	    if (nomatch || !sbptr || (sbptr == 1 && sbuf[0] == '^')) {
-		int i = zlecs;
+		int i = zlemetacs;
 		zle_setline(he);
-		zlecs = i;
+		zlemetacs = i;
 	    }
-	    ZS_memcpy(ibuf + NORM_PROMPT_POS,
+	    memcpy(ibuf + NORM_PROMPT_POS,
-		      (dir == 1) ? ZWS("fwd") : ZWS("bck"), 3);
+		   (dir == 1) ? "fwd" : "bck", 3);
 	    continue;
 	} else if(cmd == Th(z_acceptandhold)) {
-	    acceptandhold(zlenoargs);
+	    exitfn = acceptandhold;
 	    break;
 	} else if(cmd == Th(z_acceptandinfernexthistory)) {
-	    acceptandinfernexthistory(zlenoargs);
+	    exitfn = acceptandinfernexthistory;
 	    break;
 	} else if(cmd == Th(z_acceptlineanddownhistory)) {
-	    acceptlineanddownhistory(zlenoargs);
+	    exitfn = acceptlineanddownhistory;
 	    break;
 	} else if(cmd == Th(z_acceptline)) {
-	    acceptline(zlenoargs);
+	    exitfn = acceptline;
 	    break;
 	} else if(cmd == Th(z_historyincrementalsearchbackward)) {
-	    set_isrch_spot(top_spot++, hl, pos, zlecs, sbptr, dir, nomatch);
+	    set_isrch_spot(top_spot++, hl, pos, zlemetacs, sbptr, dir, nomatch);
 	    if (dir != -1)
 		dir = -1;
 	    else
 		skip_pos = 1;
 	    goto rpt;
 	} else if(cmd == Th(z_historyincrementalsearchforward)) {
-	    set_isrch_spot(top_spot++, hl, pos, zlecs, sbptr, dir, nomatch);
+	    set_isrch_spot(top_spot++, hl, pos, zlemetacs, sbptr, dir, nomatch);
 	    if (dir != 1)
 		dir = 1;
 	    else
 		skip_pos = 1;
 	    goto rpt;
 	} else if(cmd == Th(z_virevrepeatsearch)) {
-	    set_isrch_spot(top_spot++, hl, pos, zlecs, sbptr, dir, nomatch);
+	    set_isrch_spot(top_spot++, hl, pos, zlemetacs, sbptr, dir, nomatch);
 	    dir = -odir;
 	    skip_pos = 1;
 	    goto rpt;
 	} else if(cmd == Th(z_virepeatsearch)) {
-	    set_isrch_spot(top_spot++, hl, pos, zlecs, sbptr, dir, nomatch);
+	    set_isrch_spot(top_spot++, hl, pos, zlemetacs, sbptr, dir, nomatch);
 	    dir = odir;
 	    skip_pos = 1;
 	rpt:
 	    if (!sbptr && previous_search_len) {
 		if (previous_search_len > sibuf - FIRST_SEARCH_CHAR - 2) {
-		    ibuf = hrealloc((char *)ibuf, sibuf * ZLE_CHAR_SIZE,
+		    ibuf = hrealloc((char *)ibuf, sibuf,
-				    (sibuf + previous_search_len)
+				    (sibuf + previous_search_len));
-				    * ZLE_CHAR_SIZE);
 		    sbuf = ibuf + FIRST_SEARCH_CHAR;
 		    sibuf += previous_search_len;
 		}
-		ZS_memcpy(sbuf, previous_search, sbptr = previous_search_len);
+		memcpy(sbuf, previous_search, sbptr = previous_search_len);
 	    }
-	    ZS_memcpy(ibuf + NORM_PROMPT_POS,
+	    memcpy(ibuf + NORM_PROMPT_POS, (dir == 1) ? "fwd" : "bck", 3);
-		      (dir == 1) ? ZWS("fwd") : ZWS("bck"), 3);
 	    continue;
 	} else if(cmd == Th(z_viquotedinsert) ||
 	    	cmd == Th(z_quotedinsert)) {
 	    if(cmd == Th(z_viquotedinsert)) {
-		sbuf[sbptr] = ZWC('^');
+		sbuf[sbptr] = '^';
+		sbuf[sbptr+1] = '\0';
 		zrefresh();
 	    }
 	    if (getfullchar(0) == ZLEEOF)
@@ -1213,10 +1254,13 @@ doisearch(char **args, int dir)
 		feep = 1;
 		continue;
 	    }
-	    set_isrch_spot(top_spot++, hl, pos, zlecs, sbptr, dir, nomatch);
+	    set_isrch_spot(top_spot++, hl, pos, zlemetacs, sbptr, dir, nomatch);
-	    if (sbptr >= sibuf - FIRST_SEARCH_CHAR - 2) {
+	    if (sbptr >= sibuf - FIRST_SEARCH_CHAR - 2 
-		ibuf = hrealloc((char *)ibuf, sibuf * ZLE_CHAR_SIZE,
+#ifdef MULTIBYTE_SUPPORT
-				sibuf * 2 * ZLE_CHAR_SIZE);
+		- 2 * MB_CUR_MAX
+#endif
+		) {
+		ibuf = hrealloc(ibuf, sibuf, sibuf * 2);
 		sbuf = ibuf + FIRST_SEARCH_CHAR;
 		sibuf *= 2;
 	    }
@@ -1224,7 +1268,7 @@ doisearch(char **args, int dir)
 	     * We've supposedly arranged above that lastchar_wide is
 	     * always valid at this point.
 	     */
-	    sbuf[sbptr++] = LASTFULLCHAR;
+	    sbptr += zlecharasstring(LASTFULLCHAR, sbuf + sbptr);
 	}
 	if (feep)
 	    handlefeep(zlenoargs);
@@ -1232,10 +1276,13 @@ doisearch(char **args, int dir)
     }
     if (sbptr) {
 	zfree(previous_search, previous_search_len);
-	previous_search = zalloc(sbptr * ZLE_CHAR_SIZE);
+	previous_search = zalloc(sbptr);
-	ZS_memcpy(previous_search, sbuf, previous_search_len = sbptr);
+	memcpy(previous_search, sbuf, previous_search_len = sbptr);
     }
     statusline = NULL;
+    unmetafy_line();
+    if (exitfn)
+	exitfn(zlenoargs);
     selectkeymap(okeymap, 1);
     zsfree(okeymap);
     /*
@@ -1252,17 +1299,20 @@ doisearch(char **args, int dir)
 static Histent
 infernexthist(Histent he, UNUSED(char **args))
 {
+    metafy_line();
     for (he = movehistent(he, -2, HIST_FOREIGN);
 	 he; he = movehistent(he, -1, HIST_FOREIGN)) {
 	struct zle_text zt;
 	zletext(he, &zt);
 
-	if (!zlinecmp(zt.text, zt.len, zleline, zlell)) {
+	if (!zlinecmp(zt.text, zlemetaline)) {
+	    unmetafy_line();
 	    zletextfree(&zt);
 	    return movehistent(he, 1, HIST_FOREIGN);
 	}
 	zletextfree(&zt);
     }
+    unmetafy_line();
     return NULL;
 }
 
@@ -1321,7 +1371,7 @@ static int visrchsense;
 static int
 getvisrchstr(void)
 {
-    ZLE_STRING_T sbuf = zhalloc(80 * ZLE_CHAR_SIZE);
+    char *sbuf = zhalloc(80);
     int sptr = 1, ret = 0, ssbuf = 80, feep = 0;
     Thingy cmd;
     char *okeymap = ztrdup(curkeymapname);
@@ -1337,11 +1387,11 @@ getvisrchstr(void)
     }
     clearlist = 1;
     statusline = sbuf;
-    sbuf[0] = (visrchsense == -1) ? ZWC('?') : ZWC('/');
+    sbuf[0] = (visrchsense == -1) ? '?' : '/';
     selectkeymap("main", 1);
     while (sptr) {
-	sbuf[sptr] = ZWC('_');
+	sbuf[sptr] = '_';
-	statusll = sptr + 1;
+	sbuf[sptr] = '\0';
 	zrefresh();
 	if (!(cmd = getkeycmd()) || cmd == Th(z_sendbreak)) {
 	    ret = 0;
@@ -1357,32 +1407,52 @@ getvisrchstr(void)
 	    clearscreen(zlenoargs);
 	} else if(cmd == Th(z_acceptline) ||
 	    	cmd == Th(z_vicmdmode)) {
-	    int newlen;
+	    if (sptr) {
-	    sbuf[sptr] = ZWC('\0');
+		sbuf[sptr] = ZWC('\0');
-	    visrchstr = zlelineasstring(sbuf+1, sptr-1, 0, &newlen, NULL, 0);
+		visrchstr = ztrdup(sbuf+1);
-	    if (!newlen) {
+	    } else {
-	        zsfree(visrchstr);
 		visrchstr = ztrdup(vipenultsrchstr);
 	    }
 	    ret = 1;
 	    sptr = 0;
 	} else if(cmd == Th(z_backwarddeletechar) ||
-	    	cmd == Th(z_vibackwarddeletechar)) {
+		  cmd == Th(z_vibackwarddeletechar)) {
-	    sptr--;
+	    sptr = backwardmetafiedchar(sbuf+1, sbuf+sptr, NULL) - sbuf;
 	} else if(cmd == Th(z_backwardkillword) ||
-	    	cmd == Th(z_vibackwardkillword)) {
+		  cmd == Th(z_vibackwardkillword)) {
-	    while(sptr != 1 && ZC_iblank(sbuf[sptr - 1]))
+	    convchar_t cc;
-		sptr--;
+	    char *newpos;
-	    if(ZC_iident(sbuf[sptr - 1]))
+	    while (sptr != 1) {
-		while(sptr != 1 && ZC_iident(sbuf[sptr - 1]))
+		newpos = backwardmetafiedchar(sbuf+1, sbuf+sptr, &cc);
-		    sptr--;
+		if (!ZC_iblank(cc))
-	    else
+		    break;
-		while(sptr != 1 && !ZC_iident(sbuf[sptr - 1]) &&
+		sptr = newpos - sbuf;
-		      !ZC_iblank(sbuf[sptr - 1]))
+	    }
-		    sptr--;
+	    if (sptr > 1) {
+		newpos = backwardmetafiedchar(sbuf+1, sbuf+sptr, &cc);
+		if (ZC_iident(cc)) {
+		    for (;;) {
+			sptr = newpos - sbuf;
+			if (sptr == 1)
+			    break;
+			newpos = backwardmetafiedchar(sbuf+1, sbuf+sptr, &cc);
+			if (!ZC_iident(cc))
+			    break;
+		    }
+		} else {
+		    for (;;) {
+			sptr = newpos - sbuf;
+			if (sptr == 1)
+			    break;
+			newpos = backwardmetafiedchar(sbuf+1, sbuf+sptr, &cc);
+			if (ZC_iident(cc) || ZC_iblank(cc))
+			    break;
+		    }
+		}
+	    }
 	} else if(cmd == Th(z_viquotedinsert) || cmd == Th(z_quotedinsert)) {
 	    if(cmd == Th(z_viquotedinsert)) {
-		sbuf[sptr] = ZWC('^');
+		sbuf[sptr] = '^';
 		zrefresh();
 	    }
 	    if (getfullchar(0) == ZLEEOF)
@@ -1405,12 +1475,11 @@ getvisrchstr(void)
 	    }
 	  ins:
 	    if (sptr == ssbuf - 1) {
-		ZLE_STRING_T newbuf =
+		char *newbuf = (char *)zhalloc((ssbuf *= 2));
-		    (ZLE_STRING_T) zhalloc((ssbuf *= 2) * ZLE_CHAR_SIZE);
+		strcpy(newbuf, sbuf);
-		ZS_strcpy(newbuf, sbuf);
 		statusline = sbuf = newbuf;
 	    }
-	    sbuf[sptr++] = LASTFULLCHAR;
+	    sptr += zlecharasstring(LASTFULLCHAR, sbuf + sptr);
 	} else {
 	    feep = 1;
 	}
@@ -1471,8 +1540,6 @@ int
 virepeatsearch(UNUSED(char **args))
 {
     Histent he;
-    ZLE_STRING_T srcstr;
-    int srclen;
     int n = zmult;
     struct zle_text zt;
 
@@ -1482,28 +1549,26 @@ virepeatsearch(UNUSED(char **args))
 	n = -n;
 	visrchsense = -visrchsense;
     }
-    srcstr = stringaszleline(visrchstr, 0, &srclen, NULL, NULL);
     if (!(he = quietgethist(histline)))
 	return 1;
+    metafy_line();
     while ((he = movehistent(he, visrchsense, hist_skip_flags))) {
 	if (isset(HISTFINDNODUPS) && he->node.flags & HIST_DUP)
 	    continue;
 	zletext(he, &zt);
-	if (zlinecmp(zt.text, zt.len, zleline, zlell) &&
+	if (zlinecmp(zt.text, zlemetaline) &&
-	    (*visrchstr == '^'?
+	    (*visrchstr == '^' ? strpfx(zt.text, visrchstr + 1) :
-	     (zt.len == srclen - 1 &&
+	     zlinefind(zt.text, 0, visrchstr, 1, 1) != 0)) {
-	      ZS_memcmp(zt.text, srcstr + 1, zt.len) == 0) :
-	     zlinefind(zt.text, zt.len, 0, srcstr, srclen, 1, 1) != 0)) {
 	    if (--n <= 0) {
+		unmetafy_line();
 		zletextfree(&zt);
 		zle_setline(he);
-		free(srcstr);
 		return 0;
 	    }
 	}
 	zletextfree(&zt);
     }
-    free(srcstr);
+    unmetafy_line();
     return 1;
 }
 
@@ -1540,13 +1605,20 @@ historybeginningsearchbackward(char **args)
     }
     if (!(he = quietgethist(histline)))
 	return 1;
+    metafy_line();
     while ((he = movehistent(he, -1, hist_skip_flags))) {
+	int tst;
+	char sav;
 	if (isset(HISTFINDNODUPS) && he->node.flags & HIST_DUP)
 	    continue;
 	zletext(he, &zt);
-	if (zlinecmp(zt.text, zt.len, zleline, zlecs) < 0 &&
+	sav = zlemetaline[zlemetacs];
-	    zlinecmp(zt.text, zt.len, zleline, zlell)) {
+	zlemetaline[zlemetacs] = '\0';
+	tst = zlinecmp(zt.text, zlemetaline);
+	zlemetaline[zlemetacs] = sav;
+	if (tst < 0 && zlinecmp(zt.text, zlemetaline)) {
 	    if (--n <= 0) {
+		unmetafy_line();
 		zletextfree(&zt);
 		zle_setline(he);
 		zlecs = cpos;
@@ -1556,6 +1628,7 @@ historybeginningsearchbackward(char **args)
 	}
 	zletextfree(&zt);
     }
+    unmetafy_line();
     return 1;
 }
 
@@ -1580,14 +1653,20 @@ historybeginningsearchforward(char **args)
     }
     if (!(he = quietgethist(histline)))
 	return 1;
+    metafy_line();
     while ((he = movehistent(he, 1, hist_skip_flags))) {
+	char sav;
+	int tst;
 	if (isset(HISTFINDNODUPS) && he->node.flags & HIST_DUP)
 	    continue;
 	zletext(he, &zt);
-	if (zlinecmp(zt.text, zt.len, zleline, zlecs) <
+	sav = zlemetaline[zlemetacs];
-	    (he->histnum == curhist) &&
+	zlemetaline[zlemetacs] = '\0';
-	    zlinecmp(zt.text, zt.len, zleline, zlell)) {
+	tst = zlinecmp(zt.text, zlemetaline) < (he->histnum == curhist);
+	zlemetaline[zlemetacs] = sav;
+	if (tst && zlinecmp(zt.text, zlemetaline)) {
 	    if (--n <= 0) {
+		unmetafy_line();
 		zletextfree(&zt);
 		zle_setline(he);
 		zlecs = cpos;
@@ -1597,5 +1676,6 @@ historybeginningsearchforward(char **args)
 	}
 	zletextfree(&zt);
     }
+    unmetafy_line();
     return 1;
 }

Src/Zle/zle_main.c

@@ -143,12 +143,10 @@ mod_export int lastcmd;
 /**/
 mod_export Widget compwidget;
 
-/* the status line, and its length */
+/* the status line, a null-terminated metafied string */
 
 /**/
-mod_export ZLE_STRING_T statusline;
+mod_export char *statusline;
-/**/
-mod_export int statusll;
 
 /* The current history line and cursor position for the top line *
  * on the buffer stack.                                          */
@@ -1240,12 +1238,16 @@ zleread(char **lp, char **rp, int flags, int context)
 int
 execzlefunc(Thingy func, char **args, int set_bindk)
 {
-    int r = 0, ret = 0;
+    int r = 0, ret = 0, remetafy = 0;
     Widget w;
     Thingy save_bindk = bindk;
 
     if (set_bindk)
 	bindk = func;
+    if (zlemetaline) {
+	unmetafy_line();
+	remetafy = 1;
+    }
 
     if(func->flags & DISABLED) {
 	/* this thingy is not the name of a widget */
@@ -1350,6 +1352,8 @@ execzlefunc(Thingy func, char **args, int set_bindk)
      * directly.
      */
     CCRIGHT();
+    if (remetafy)
+	metafy_line();
     return ret;
 }
 
@@ -1632,8 +1636,7 @@ describekeybriefly(UNUSED(char **args))
     if (statusline)
 	return 1;
     clearlist = 1;
-    statusline = ZWS("Describe key briefly: _");
+    statusline = "Describe key briefly: _";
-    statusll = ZS_strlen(statusline);
     zrefresh();
     seq = getkeymapcmd(curkeymap, &func, &str);
     statusline = NULL;

Src/Zle/zle_misc.c

@@ -914,24 +914,27 @@ executenamedcommand(char *prmt)
     Thingy cmd;
     int l, len, feep = 0, listed = 0, curlist = 0;
     int ols = (listshown && validlist), olll = lastlistlen;
-    ZLE_STRING_T cmdbuf, ptr, zprmt;
+    char *cmdbuf, *ptr;
     char *okeymap = ztrdup(curkeymapname);
 
     clearlist = 1;
     /* prmt may be constant */
     prmt = ztrdup(prmt);
-    zprmt = stringaszleline(prmt, 0, &l, NULL, NULL);
+    l = strlen(prmt);
-    cmdbuf = zhalloc((l + NAMLEN + 2) * ZLE_CHAR_SIZE);
+    cmdbuf = (char *)zhalloc(l + NAMLEN + 2 +
-    ZS_memcpy(cmdbuf, zprmt, l);
+#ifdef MULTIBYTE_SUPPORT
-    free(zprmt);
+			     2 * MB_CUR_MAX
+#endif
+			     );
+    strcpy(cmdbuf, prmt);
     zsfree(prmt);
     statusline = cmdbuf;
     selectkeymap("main", 1);
     ptr = cmdbuf += l;
     len = 0;
     for (;;) {
-	*ptr = ZWC('_');
+	*ptr = '_';
-	statusll = l + len + 1;
+	ptr[1] = '\0';
 	zrefresh();
 	if (!(cmd = getkeycmd()) || cmd == Th(z_sendbreak)) {
 	    statusline = NULL;
@@ -966,31 +969,45 @@ executenamedcommand(char *prmt)
 		zmult = zmultsav;
 	    }
 	} else if(cmd == Th(z_viquotedinsert)) {
-	    *ptr = ZWC('^');
+	    *ptr = '^';
 	    zrefresh();
 	    getfullchar(0);
-	    if(LASTFULLCHAR == ZLEEOF || !LASTFULLCHAR || len == NAMLEN)
+	    if(LASTFULLCHAR == ZLEEOF || !LASTFULLCHAR || len >= NAMLEN)
 		feep = 1;
 	    else {
-		*ptr++ = LASTFULLCHAR, len++, curlist = 0;
+		int ret = zlecharasstring(LASTFULLCHAR, ptr);
+		len += ret;
+		ptr += ret;
+		curlist = 0;
 	    }
 	} else if(cmd == Th(z_quotedinsert)) {
 	    if(getfullchar(0) == ZLEEOF ||
 	       !LASTFULLCHAR || len == NAMLEN)
 		feep = 1;
 	    else {
-		*ptr++ = LASTFULLCHAR, len++, curlist = 0;
+		int ret = zlecharasstring(LASTFULLCHAR, ptr);
+		len += ret;
+		ptr += ret;
+		curlist = 0;
 	    }
 	} else if(cmd == Th(z_backwarddeletechar) ||
-	    	cmd == Th(z_vibackwarddeletechar)) {
+		  cmd == Th(z_vibackwarddeletechar)) {
 	    if (len) {
-		len--, ptr--, curlist = 0;
+		ptr = backwardmetafiedchar(cmdbuf, ptr, NULL);
+		len = ptr - cmdbuf;
+		curlist = 0;
 	    }
 	} else if(cmd == Th(z_killregion) || cmd == Th(z_backwardkillword) ||
 		  cmd == Th(z_vibackwardkillword)) {
 	    if (len)
 		curlist = 0;
-	    while (len && (len--, *--ptr != ZWC('-')));
+	    while (len) {
+		convchar_t cc;
+		ptr = backwardmetafiedchar(cmdbuf, ptr, &cc);
+		len = ptr - cmdbuf;
+		if (cc == ZWC('-'))
+		    break;
+	    }
 	} else if(cmd == Th(z_killwholeline) || cmd == Th(z_vikillline) ||
 	    	cmd == Th(z_backwardkillline)) {
 	    len = 0;
@@ -1003,10 +1020,7 @@ executenamedcommand(char *prmt)
 		Thingy r;
 		unambiguous:
 		*ptr = 0;
-		namedcmdstr = zlelineasstring(cmdbuf, len, 0, NULL, NULL, 0);
+		r = rthingy(cmdbuf);
-		r = rthingy(namedcmdstr);
-		free(namedcmdstr);
-		namedcmdstr = NULL;
 		if (!(r->flags & DISABLED)) {
 		    unrefthingy(r);
 		    statusline = NULL;
@@ -1033,9 +1047,9 @@ executenamedcommand(char *prmt)
 
 		namedcmdll = newlinklist();
 
-		namedcmdstr = zlelineasstring(cmdbuf, len, 0, NULL, NULL, 0);
+		*ptr = '\0';
+		namedcmdstr = cmdbuf;
 		scanhashtable(thingytab, 1, 0, DISABLED, scancompcmd, 0);
-		free(namedcmdstr);
 		namedcmdstr = NULL;
 
 		if (empty(namedcmdll)) {
@@ -1046,39 +1060,29 @@ executenamedcommand(char *prmt)
 		} else if (cmd == Th(z_listchoices) ||
 		    cmd == Th(z_deletecharorlist)) {
 		    int zmultsav = zmult;
-		    *ptr = ZWC('_');
+		    *ptr = '_';
-		    statusll = l + len + 1;
+		    ptr[1] = '\0';
 		    zmult = 1;
 		    listlist(namedcmdll);
 		    listed = curlist = 1;
 		    showinglist = 0;
 		    zmult = zmultsav;
 		} else if (!nextnode(firstnode(namedcmdll))) {
-		    char *peekstr = ztrdup(peekfirst(namedcmdll));
+		    strcpy(ptr = cmdbuf, peekfirst(namedcmdll));
-		    ZLE_STRING_T ztmp = stringaszleline(peekstr, 0, &len,
+		    len = strlen(ptr);
-							NULL, NULL);
-		    zsfree(peekstr);
-		    ZS_memcpy(ptr = cmdbuf, ztmp, len);
 		    ptr += len;
-		    free(ztmp);
+		    if (cmd == Th(z_acceptline) || cmd == Th(z_vicmdmode))
-		    if(cmd == Th(z_acceptline) || cmd == Th(z_vicmdmode))
 			goto unambiguous;
 		} else {
-		    int ltmp;
+		    strcpy(cmdbuf, peekfirst(namedcmdll));
-		    char *peekstr = ztrdup(peekfirst(namedcmdll));
-		    ZLE_STRING_T ztmp = stringaszleline(peekstr, 0, &ltmp,
-							NULL, NULL);
-		    zsfree(peekstr);
-		    ZS_memcpy(cmdbuf, ztmp, ltmp);
-		    free(ztmp);
 		    ptr = cmdbuf + namedcmdambig;
-		    *ptr = ZWC('_');
+		    *ptr = '_';
+		    ptr[1] = '\0';
 		    if (isset(AUTOLIST) &&
 			!(isset(LISTAMBIGUOUS) && namedcmdambig > len)) {
 			int zmultsav = zmult;
 			if (isset(LISTBEEP))
 			    feep = 1;
-			statusll = l + namedcmdambig + 1;
 			zmult = 1;
 			listlist(namedcmdll);
 			listed = curlist = 1;
@@ -1100,8 +1104,16 @@ executenamedcommand(char *prmt)
 #endif
 		    if (ZC_icntrl(LASTFULLCHAR))
 			feep = 1;
-		    else
+		    else {
-			*ptr++ = LASTFULLCHAR, len++, curlist = 0;
+			int ret = zlecharasstring(LASTFULLCHAR, ptr);
+			len += ret;
+			ptr += ret;
+			if (listed) {
+			    clearlist = listshown = 1;
+			    listed = 0;
+			} else
+			    curlist = 0;
+		    }
 		}
 	    }
 	}

Src/Zle/zle_move.c

@@ -158,6 +158,144 @@ decpos(int *pos)
 }
 #endif
 
+
+/* Size of buffer in the following function */
+#define BMC_BUFSIZE MB_CUR_MAX
+/*
+ * For a metafied string that starts at "start" and where the
+ * current position is "ptr", go back one full character,
+ * taking account of combining characters if necessary.
+ */
+
+/**/
+char *
+backwardmetafiedchar(char *start, char *ptr, convchar_t *retchr)
+{
+#ifdef MULTIBYTE_SUPPORT
+    int charlen = 0;
+    char *last = NULL, *bufptr, *endptr = ptr;
+    convchar_t lastc;
+    mbstate_t mbs;
+    size_t ret;
+    wchar_t wc;
+    VARARR(char, buf, BMC_BUFSIZE);
+
+    bufptr = buf + BMC_BUFSIZE;
+    while (ptr > start) {
+	ptr--;
+	/*
+	 * Scanning backwards we're not guaranteed ever to find a
+	 * valid character.  If we've looked as far as we should
+	 * need to, give up.
+	 */
+	if (bufptr-- == buf)
+	    break;
+	charlen++;
+	if (ptr > start && ptr[-1] == Meta)
+	    *bufptr = *ptr-- ^ 32;
+	else
+	    *bufptr = *ptr;
+
+	/* we always need to restart the character from scratch */
+	memset(&mbs, 0, sizeof(mbs));
+	ret = mbrtowc(&wc, bufptr, charlen, &mbs);
+	if (ret == 0) {
+	    /* NULL: unlikely, but handle anyway. */
+	    if (last) {
+		if (retchr)
+		    *retchr = lastc;
+		return last;
+	    } else {
+		if (retchr)
+		    *retchr = wc;
+		return ptr;
+	    }
+	}
+	if (ret >= 0) {
+	    if (ret < charlen) {
+		/* The last character didn't convert, so use it raw. */
+		break;
+	    }
+	    if (!isset(COMBININGCHARS)) {
+		if (retchr)
+		    *retchr = wc;
+		return ptr;
+	    }
+ 	    /* HERE: test for combining char, fix when test changes */
+	    if (!iswpunct(wc) || wcwidth(wc) != 0) {
+		/* not a combining character... */
+		if (last) {
+		    /*
+		     * ... but we were looking for a suitable base character,
+		     * test it.
+		     */
+		    /* HERE this test will change too */
+		    if (iwsalnum(wc) && wcwidth(wc) > 0) {
+			/*
+			 * Yes, this will do.
+			 */
+			if (retchr)
+			    *retchr = wc;
+			return ptr;
+		    } else {
+			/* No, just return the first character we found */
+			if (retchr)
+			    *retchr = lastc;
+			return last;
+		    }
+		}
+		/* This is the first character, so just return it. */
+		if (retchr)
+		    *retchr = wc;
+		return ptr;    
+	    }
+	    if (!last) {
+		/* still looking for the character immediately before ptr */
+		last = ptr;
+	    }
+	    /* searching for base character of combining character */
+	    charlen = 0;
+	    bufptr = buf + BMC_BUFSIZE;
+	}
+	/*
+	 * Else keep scanning this character even if MB_INVALID:  we can't
+	 * expect MB_INCOMPLETE to work when moving backwards.
+	 */
+    }
+    /*
+     * Found something we didn't like, was there a good character
+     * immediately before ptr?
+     */
+    if (last) {
+	if (retchr)
+	    *retchr = lastc;
+	return last;
+    }
+    /*
+     * No, we couldn't find any good character, so just treat
+     * the last unmetafied byte we found as a character.
+     */
+#endif
+    if (endptr > start) {
+	if (endptr > start - 1 && endptr[-2] == Meta)
+	{
+	    if (retchr)
+		*retchr = (convchar_t)(endptr[-1] ^ 32);
+	    return endptr - 2;
+	}
+	else
+	{
+	    if (retchr)
+		*retchr = (convchar_t)endptr[-1];
+	    return endptr - 1;
+	}
+    }
+    if (retchr)
+	*retchr = (convchar_t)0;
+    return endptr;
+}
+
+
 /**/
 int
 beginningofline(char **args)

Src/Zle/zle_refresh.c

@@ -1387,11 +1387,15 @@ zrefresh(void)
 	more_end = 1;
 
     if (statusline) {
+	int outll, outsz;
+	ZLE_STRING_T outputline =
+	    stringaszleline(statusline, 0, &outll, &outsz, NULL); 
+
 	rpms.tosln = rpms.ln + 1;
 	nbuf[rpms.ln][winw + 1] = zr_zr;	/* text not wrapped */
 	snextline(&rpms);
-	u = statusline;
+	u = outputline;
-	for (; u < statusline + statusll; u++) {
+	for (; u < outputline + outll; u++) {
 #ifdef MULTIBYTE_SUPPORT
 	    if (iswprint(*u)) {
 		int width = wcwidth(*u);
@@ -1449,6 +1453,7 @@ zrefresh(void)
 	     */
 	    snextline(&rpms);
 	}
+	zfree(outputline, outsz);
     }
     *rpms.s = zr_zr;
 

Src/Zle/zle_thingy.c

@@ -406,16 +406,15 @@ bin_zle_list(UNUSED(char *name), char **args, Options ops, UNUSED(char func))
 static int
 bin_zle_refresh(UNUSED(char *name), char **args, Options ops, UNUSED(char func))
 {
-    ZLE_STRING_T s = statusline;
+    char *s = statusline;
-    int sl = statusll, ocl = clearlist;
+    int ocl = clearlist;
 
     if (!zleactive)
 	return 1;
     statusline = NULL;
-    statusll = 0;
     if (*args) {
 	if (**args)
-	    statusline = stringaszleline(*args, 0, &statusll, NULL, NULL);
+	    statusline = *args;
 	if (*++args) {
 	    LinkList l = newlinklist();
 	    int zmultsav = zmult;
@@ -439,12 +438,8 @@ bin_zle_refresh(UNUSED(char *name), char **args, Options ops, UNUSED(char func))
     }
     zrefresh();
 
-    if (statusline)
-	free(statusline);
-
     clearlist = ocl;
     statusline = s;
-    statusll = sl;
     return 0;
 }
 

Src/Zle/zle_utils.c

@@ -105,6 +105,54 @@ zleaddtoline(int chr)
     zlemetaline[zlemetacs++] = chr;
 }
 
+/*
+ * Convert a line editor character to a possibly multibyte character
+ * in a metafied string.  To be safe buf should have space for at least
+ * 2 * MB_CUR_MAX chars for multibyte mode and 2 otherwise.  Returns the
+ * length of the string added.
+ */
+
+/**/
+int
+zlecharasstring(ZLE_CHAR_T inchar, char *buf)
+{
+#ifdef MULTIBYTE_SUPPORT
+    size_t ret;
+    char *ptr;
+
+    ret = wctomb(buf, inchar);
+    if (ret <= 0) {
+	/* Ick. */
+	buf[0] = '?';
+	return 1;
+    }
+    ptr = buf + ret - 1;
+    for (;;) {
+	if (imeta(*ptr)) {
+	    char *ptr2 = buf + ret - 1;
+	    for (;;) {
+		ptr2[1] = ptr2[0];
+		if (ptr2 == ptr)
+		    break;
+		ptr2--;
+	    }
+	    *ptr = Meta;
+	    ret++;
+	}
+
+	if (ptr == buf)
+	    return ret;
+	ptr--;
+    }
+#else
+    if (imeta(inchar)) {
+	buf[0] = Meta;
+	buf[1] = inchar ^ 32;
+    } else
+	buf[0] = inchar;
+#endif
+}
+
 /*
  * Input a line in internal zle format, possibly using wide characters,
  * possibly not, together with its length and the cursor position.
@@ -621,7 +669,7 @@ void
 setline(char *s, int flags)
 {
     char *scp;
-
+	
     if (flags & ZSL_COPY)
 	scp = ztrdup(s);
     else
@@ -639,7 +687,6 @@ setline(char *s, int flags)
     else if (zlecs > zlell)
 	zlecs = zlell;
     CCRIGHT();
-
     if (flags & ZSL_COPY)
 	free(scp);
 }

Src/Zle/zle_vi.c

@@ -853,8 +853,7 @@ vicapslockpanic(UNUSED(char **args))
 {
     clearlist = 1;
     zbeep();
-    statusline = ZWS("press a lowercase key to continue");
+    statusline = "press a lowercase key to continue";
-    statusll = ZS_strlen(statusline);
     zrefresh();
     while (!ZC_ilower(getfullchar(0)));
     statusline = NULL;

Src/Zle/zle_word.c

@@ -185,7 +185,6 @@ viforwardwordend(char **args)
 	return ret;
     }
     while (n--) {
-	/* HERE: the zlecs + 1 here is suspect */
 	int pos;
 	while (zlecs != zlell) {
 	    pos = zlecs;

Src/utils.c

@@ -3967,6 +3967,50 @@ nicedup(const char *s, int heap)
 }
 
 
+/*
+ * The guts of mb_metacharlenconv().  This version assumes we are
+ * processing a true multibyte character string without tokens, and
+ * takes the shift state as an argument.
+ */
+
+/**/
+mod_export int
+mb_metacharlenconv_r(const char *s, wint_t *wcp, mbstate_t *mbsp)
+{
+    size_t ret = MB_INVALID;
+    char inchar;
+    const char *ptr;
+    wchar_t wc;
+
+    for (ptr = s; *ptr; ) {
+	if (*ptr == Meta) {
+	    inchar = *++ptr ^ 32;
+	    DPUTS(!*ptr,
+		  "BUG: unexpected end of string in mb_metacharlen()\n");
+	} else
+	    inchar = *ptr;
+	ptr++;
+	ret = mbrtowc(&wc, &inchar, 1, mbsp);
+
+	if (ret == MB_INVALID)
+	    break;
+	if (ret == MB_INCOMPLETE)
+	    continue;
+	if (wcp)
+	    *wcp = wc;
+	return ptr - s;
+    }
+
+    if (wcp)
+	*wcp = WEOF;
+    /* No valid multibyte sequence */
+    memset(mbsp, 0, sizeof(*mbsp));
+    if (ptr > s) {
+	return 1 + (*s == Meta);	/* Treat as single byte character */
+    } else
+	return 0;		/* Probably shouldn't happen */
+}
+
 /*
  * Length of metafied string s which contains the next multibyte
  * character; single (possibly metafied) character if string is not null
@@ -3982,11 +4026,6 @@ nicedup(const char *s, int heap)
 mod_export int
 mb_metacharlenconv(const char *s, wint_t *wcp)
 {
-    char inchar;
-    const char *ptr;
-    size_t ret;
-    wchar_t wc;
-
     if (!isset(MULTIBYTE)) {
 	/* treat as single byte, possibly metafied */
 	if (wcp)
@@ -4009,37 +4048,7 @@ mb_metacharlenconv(const char *s, wint_t *wcp)
 	return 1;
     }
 
-    ret = MB_INVALID;
+    return mb_metacharlenconv_r(s, wcp, &mb_shiftstate);
-    for (ptr = s; *ptr; ) {
-	if (*ptr == Meta) {
-	    inchar = *++ptr ^ 32;
-#ifdef DEBUG
-	    if (!*ptr)
-		fprintf(stderr,
-			"BUG: unexpected end of string in mb_metacharlen()\n");
-#endif
-	} else
-	    inchar = *ptr;
-	ptr++;
-	ret = mbrtowc(&wc, &inchar, 1, &mb_shiftstate);
-
-	if (ret == MB_INVALID)
-	    break;
-	if (ret == MB_INCOMPLETE)
-	    continue;
-	if (wcp)
-	    *wcp = wc;
-	return ptr - s;
-    }
-
-    if (wcp)
-	*wcp = WEOF;
-    /* No valid multibyte sequence */
-    memset(&mb_shiftstate, 0, sizeof(mb_shiftstate));
-    if (ptr > s) {
-	return 1 + (*s == Meta);	/* Treat as single byte character */
-    } else
-	return 0;		/* Probably shouldn't happen */
 }
 
 /*

Src/zsh.h

@@ -1606,12 +1606,11 @@ struct histent {
 
     Histent up;			/* previous line (moving upward)    */
     Histent down;		/* next line (moving downward)      */
-#ifdef MULTIBYTE_SUPPORT	/* (Note: must match ZLE_STRING_T!) */
+    char *zle_text;		/* the edited history line,
-    wchar_t *zle_text;		/* the edited history line          */
+				 * a metafied, NULL-terminated string,
-#else
+				 * i.e the same format as the original
-    char *zle_text;		/* the edited history line          */
+				 * entry
-#endif
+				 */
-    int zle_len;		/* length of zle_text */
     time_t stim;		/* command started time (datestamp) */
     time_t ftim;		/* command finished time            */
     short *words;		/* Position of words in history     */