mirrors/zsh
1
0
Fork 0
mirror of git://git.code.sf.net/p/zsh/code synced 2025-10-31 06:00:54 +01:00
Code Releases Activity

28528: Rearrange zleparse to lexflags

Browse source 
Add (z+n+)
This commit is contained in:
Peter Stephenson 2010-12-14 09:59:04 +00:00
parent 5858e79f4d
commit eab4f9a83c
10 changed files with 112 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

10
ChangeLog
View file

@ -1,3 +1,11 @@
2010-12-14 Peter Stephenson <pws@csr.com>
* 28528: Doc/Zsh/expn.yo, Src/hist.c, Src/lex.c, Src/subst.c,
Src/zsh.h, Src/Zle/compcore.c, Src/Zle/compctl.c,
Src/Zle/zle_tricky.c, Test/D04parameter.ztst: clear up use of
zleparse variable into lexflags; add (z+n+) for splitting with
newline treated as ordinary whitespace.
2010-12-13 Peter Stephenson <pws@csr.com>
* 28526: Src/lex.c, Test/D04parameter.ztst: zplitting
@ -13942,5 +13950,5 @@
*****************************************************
* This is used by the shell to define $ZSH_PATCHLEVEL
* $Revision: 1.5148 $
* $Revision: 1.5149 $
*****************************************************

11
Doc/Zsh/expn.yo
View file

@ -1009,14 +1009,17 @@ find the words, i.e. taking into account any quoting in the value.
Comments are not treated specially but as ordinary strings, similar
to interactive shells with the tt(INTERACTIVE_COMMENTS) option unset.
The flag can take option letters between a following pair of
`tt(PLUS())' characters. tt(LPAR()z+PLUS()c+PLUS()RPAR()) causes
comments to be parsed as a string and retained; any field in the
The flag can take a combination of option letters between a following
pair of `tt(PLUS())' characters. tt(LPAR()z+PLUS()c+PLUS()RPAR())
causes comments to be parsed as a string and retained; any field in the
resulting array beginning with an unquoted comment character is a
comment. tt(LPAR()z+PLUS()C+PLUS()RPAR()) causes comments to be parsed
and removed. The rule for comments is standard: anything between a word
starting with the third charcter of tt($HISTCHARS), default tt(#), up to
the next newline is a comment.
the next newline is a comment. tt(LPAR()z+PLUS()n+PLUS()RPAR()) causes
unquoted newlines to be treated as ordinary whitespace, else they are
treated as if they are shell code delimiters and converted to
semicolons.
Note that this is done very late, as for the `tt((s))' flag. So to
access single words in the result, one has to use nested expansions as

6
Src/Zle/compcore.c
View file

@ -1481,7 +1481,7 @@ set_comp_sep(void)
/* Put the string in the lexer buffer and call the lexer to *
* get the words we have to expand. */
zleparse = 1;
lexflags = LEXFLAGS_ACTIVE;
ocs = zlemetacs;
oll = zlemetall;
ol = zlemetaline;
@ -1616,7 +1616,7 @@ set_comp_sep(void)
}
else
p = NULL;
if (!got && !zleparse) {
if (!got && !lexflags) {
DPUTS(!p, "no current word in substr");
got = 1;
cur = i;
@ -1634,7 +1634,7 @@ set_comp_sep(void)
noaliases = ona;
strinend();
inpop();
errflag = zleparse = 0;
errflag = lexflags = 0;
noerrs = ne;
lexrestore();
wb = owb;

10
Src/Zle/compctl.c
View file

@ -2789,7 +2789,7 @@ sep_comp_string(char *ss, char *s, int noffs)
/* Put the string in the lexer buffer and call the lexer to *
* get the words we have to expand. */
zleparse = 1;
lexflags = LEXFLAGS_ACTIVE;
addedx = 1;
noerrs = 1;
lexsave();
@ -2828,7 +2828,7 @@ sep_comp_string(char *ss, char *s, int noffs)
addlinknode(foo, (p = ztrdup(tokstr)));
else
p = NULL;
if (!got && !zleparse) {
if (!got && !lexflags) {
DPUTS(!p, "no current word in substr");
got = 1;
cur = i;
@ -2843,7 +2843,7 @@ sep_comp_string(char *ss, char *s, int noffs)
noaliases = ona;
strinend();
inpop();
errflag = zleparse = 0;
errflag = lexflags = 0;
noerrs = ne;
lexrestore();
wb = owb;
@ -3703,7 +3703,7 @@ makecomplistflags(Compctl cc, char *s, int incmd, int compadd)
/* Put the string in the lexer buffer and call the lexer to *
* get the words we have to expand. */
zleparse = 1;
lexflags = LEXFLAGS_ACTIVE;
lexsave();
tmpbuf = (char *)zhalloc(strlen(cc->str) + 5);
sprintf(tmpbuf, "foo %s", cc->str); /* KLUDGE! */
@ -3721,7 +3721,7 @@ makecomplistflags(Compctl cc, char *s, int incmd, int compadd)
noaliases = ona;
strinend();
inpop();
errflag = zleparse = 0;
errflag = lexflags = 0;
lexrestore();
/* Fine, now do full expansion. */
prefork(foo, 0);

16
Src/Zle/zle_tricky.c
View file

@ -1140,7 +1140,7 @@ get_comp_string(void)
zsfree(varname);
varname = NULL;
insubscr = 0;
zleparse = 1;
lexflags = LEXFLAGS_ACTIVE;
clwpos = -1;
lexsave();
inpush(dupstrspace(linptr), 0, NULL);
@ -1244,7 +1244,7 @@ get_comp_string(void)
if (wordpos != redirpos)
wordpos = redirpos = 0;
}
if (!zleparse && !tt0) {
if (!lexflags && !tt0) {
/* This is done when the lexer reached the word the cursor is on. */
tt = tokstr ? dupstring(tokstr) : NULL;
@ -1345,7 +1345,7 @@ get_comp_string(void)
(sl - 1) : (zlemetacs_qsub - wb)]);
}
} while (tok != LEXERR && tok != ENDINPUT &&
(tok != SEPER || (zleparse && !tt0)));
(tok != SEPER || (lexflags && !tt0)));
/* Calculate the number of words stored in the clwords array. */
clwnum = (tt || !wordpos) ? wordpos : wordpos - 1;
zsfree(clwords[clwnum]);
@ -1360,7 +1360,7 @@ get_comp_string(void)
}
strinend();
inpop();
errflag = zleparse = 0;
errflag = lexflags = 0;
if (parbegin != -1) {
/* We are in command or process substitution if we are not in
* a $((...)). */
@ -2707,7 +2707,7 @@ doexpandhist(void)
noaliases = ona;
strinend();
inpop();
zleparse = 0;
lexflags = 0;
lexrestore();
expanding = 0;
@ -2807,7 +2807,7 @@ getcurcmd(void)
int curlincmd;
char *s = NULL;
zleparse = 2;
lexflags = LEXFLAGS_ACTIVE;
lexsave();
metafy_line();
inpush(dupstrspace(zlemetaline), 0, NULL);
@ -2825,11 +2825,11 @@ getcurcmd(void)
cmdwe = zlemetall + 1 - inbufct;
}
}
while (tok != ENDINPUT && tok != LEXERR && zleparse);
while (tok != ENDINPUT && tok != LEXERR && lexflags);
popheap();
strinend();
inpop();
errflag = zleparse = 0;
errflag = lexflags = 0;
unmetafy_line();
lexrestore();

25
Src/hist.c
View file

@ -2345,7 +2345,8 @@ readhistfile(char *fn, int err, int readflags)
/*
* Attempt to do this using the lexer.
*/
LinkList wordlist = bufferwords(NULL, pt, NULL, 1);
LinkList wordlist = bufferwords(NULL, pt, NULL,
LEXFLAGS_COMMENTS_KEEP);
LinkNode wordnode;
int nwords_max;
nwords_max = 2 * countlinknodes(wordlist);
@ -2905,10 +2906,10 @@ histfileIsLocked(void)
/**/
mod_export LinkList
bufferwords(LinkList list, char *buf, int *index, int comments)
bufferwords(LinkList list, char *buf, int *index, int flags)
{
int num = 0, cur = -1, got = 0, ne = noerrs;
int owb = wb, owe = we, oadx = addedx, ozp = zleparse, onc = nocomments;
int owb = wb, owe = we, oadx = addedx, ozp = lexflags, onc = nocomments;
int ona = noaliases, ocs = zlemetacs, oll = zlemetall;
int forloop = 0, rcquotes = opts[RCQUOTES];
char *p, *addedspaceptr;
@ -2925,6 +2926,12 @@ bufferwords(LinkList list, char *buf, int *index, int comments)
addedx = 0;
noerrs = 1;
lexsave();
lexflags = flags | LEXFLAGS_ACTIVE;
/*
* Are we handling comments?
*/
nocomments = !(flags & (LEXFLAGS_COMMENTS_KEEP|
LEXFLAGS_COMMENTS_STRIP));
if (buf) {
int l = strlen(buf);
@ -2943,18 +2950,10 @@ bufferwords(LinkList list, char *buf, int *index, int comments)
inpush(p, 0, NULL);
zlemetall = strlen(p) ;
zlemetacs = zlemetall + 1;
/*
* If comments is non-zero we are handling comments.
* zleparse indicates the mode to the lexer.
*/
zleparse = 1 + comments;
nocomments = !comments;
} else {
int ll, cs;
char *linein;
zleparse = 1;
linein = zleentry(ZLE_CMD_GET_LINE, &ll, &cs);
zlemetall = ll + 1; /* length of line plus space added below */
zlemetacs = cs;
@ -3096,7 +3095,7 @@ bufferwords(LinkList list, char *buf, int *index, int comments)
}
forloop--;
}
if (!got && !zleparse) {
if (!got && !lexflags) {
got = 1;
cur = num - 1;
}
@ -3121,7 +3120,7 @@ bufferwords(LinkList list, char *buf, int *index, int comments)
strinend();
inpop();
errflag = 0;
zleparse = ozp;
lexflags = ozp;
nocomments = onc;
noerrs = ne;
lexrestore();

45
Src/lex.c
View file

@ -117,15 +117,11 @@ mod_export int wb, we;
mod_export int noaliases;
/*
* we are parsing a line sent to use by the editor, or some other string
* that's not part of standard command input (e.g. eval is part of
* normal command input).
* If non-zero, we are parsing a line sent to use by the editor, or some
* other string that's not part of standard command input (e.g. eval is
* part of normal command input).
*
* zleparse = 1 is the normal case.
* zleparse = 2 is used for word splitting; the difference is we
* preserve comments.
* zleparse = 3 is also for word splitting, here handling comments
* but stripping them.
* Set of bits from LEXFLAGS_*.
*
* Note that although it is passed into the lexer as an input, the
* lexer can set it to zero after finding the word it's searching for.
@ -134,7 +130,7 @@ mod_export int noaliases;
*/
/**/
mod_export int zleparse;
mod_export int lexflags;
/**/
mod_export int wordbeg;
@ -429,7 +425,7 @@ zshlex(void)
isnewlin = 0;
else
isnewlin = (inbufct) ? -1 : 1;
if (tok == SEMI || tok == NEWLIN)
if (tok == SEMI || (tok == NEWLIN && !(lexflags & LEXFLAGS_NEWLINE)))
tok = SEPER;
}
@ -588,9 +584,9 @@ add(int c)
}
}
#define SETPARBEGIN {if (zleparse && !(inbufflags & INP_ALIAS) && zlemetacs >= zlemetall+1-inbufct) parbegin = inbufct;}
#define SETPARBEGIN {if (lexflags && !(inbufflags & INP_ALIAS) && zlemetacs >= zlemetall+1-inbufct) parbegin = inbufct;}
#define SETPAREND {\
if (zleparse && !(inbufflags & INP_ALIAS) && parbegin != -1 && parend == -1) {\
if (lexflags && !(inbufflags & INP_ALIAS) && parbegin != -1 && parend == -1) {\
if (zlemetacs >= zlemetall + 1 - inbufct)\
parbegin = -1;\
else\
@ -760,22 +756,17 @@ gettok(void)
/*
* Handle comments. There are some special cases when this
* is not normal command input: zleparse implies we are examining
* is not normal command input: lexflags implies we are examining
* a line lexically without it being used for normal command input.
* If zleparse is 1 we treat comments as normal for interactive
* mode.
* If zleparse is 2 (which has actually got nothing to do with zle)
* we always handle comments and retain them.
* If zleparse is 3 we always handle comments and discard them.
*/
if (c == hashchar && !nocomments &&
(isset(INTERACTIVECOMMENTS) ||
((zleparse != 1) && !expanding &&
((!lexflags || (lexflags & LEXFLAGS_COMMENTS)) && !expanding &&
(!interact || unset(SHINSTDIN) || strin)))) {
/* History is handled here to prevent extra *
* newlines being inserted into the history. */
if (zleparse == 2) {
if (lexflags & LEXFLAGS_COMMENTS_KEEP) {
len = 0;
bptr = tokstr = (char *)hcalloc(bsiz = 32);
add(c);
@ -783,14 +774,14 @@ gettok(void)
while ((c = ingetc()) != '\n' && !lexstop) {
hwaddc(c);
addtoline(c);
if (zleparse == 2)
if (lexflags & LEXFLAGS_COMMENTS_KEEP)
add(c);
}
if (errflag)
peek = LEXERR;
else {
if (zleparse == 2) {
if (lexflags & LEXFLAGS_COMMENTS_KEEP) {
*bptr = '\0';
if (!lexstop)
hungetc(c);
@ -805,7 +796,7 @@ gettok(void)
* we don't want a newline token since it's
* treated specially.
*/
if (zleparse == 3 && lexstop)
if ((lexflags & LEXFLAGS_COMMENTS_STRIP) && lexstop)
peek = ENDINPUT;
else
peek = NEWLIN;
@ -1778,7 +1769,7 @@ gotword(void)
we = zlemetall + 1 - inbufct + (addedx == 2 ? 1 : 0);
if (zlemetacs <= we) {
wb = zlemetall - wordbeg + addedx;
zleparse = 0;
lexflags = 0;
}
}
@ -1813,11 +1804,11 @@ exalias(void)
} else
zshlextext = tokstr;
if (zleparse && !(inbufflags & INP_ALIAS)) {
int zp = zleparse;
if (lexflags && !(inbufflags & INP_ALIAS)) {
int zp = lexflags;
gotword();
if (zp == 1 && !zleparse) {
if (zp == 1 && !lexflags) {
if (zshlextext == copy)
zshlextext = tokstr;
return 0;

19
Src/subst.c
View file

@ -1557,9 +1557,7 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
* spbreak, see above; fairly straighforward in use but c.f.
* the comment for mods.
*
* This ultimately becomes zleparse during lexical analysis, via
* the comments argument to bufferwords(). It's got nothing
* to do with zle.
* This gets set to one of the LEXFLAGS_* values.
*/
int shsplit = 0;
/*
@ -1937,19 +1935,24 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
break;
case 'z':
shsplit = 1;
shsplit = LEXFLAGS_ACTIVE;
if (s[1] == '+') {
s += 2;
while (*s && *s != '+' && *s != ')' && *s != Outpar) {
switch (*s++) {
case 'c':
/* Parse and keep comments */
shsplit = 2;
shsplit |= LEXFLAGS_COMMENTS_KEEP;
break;
case 'C':
/* Parse and remove comments */
shsplit = 3;
shsplit |= LEXFLAGS_COMMENTS_STRIP;
break;
case 'n':
/* Treat newlines as whitespace */
shsplit |= LEXFLAGS_NEWLINE;
break;
default:
@ -3232,10 +3235,10 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
if (isarr) {
char **ap;
for (ap = aval; *ap; ap++)
list = bufferwords(list, *ap, NULL, shsplit-1);
list = bufferwords(list, *ap, NULL, shsplit);
isarr = 0;
} else
list = bufferwords(NULL, val, NULL, shsplit-1);
list = bufferwords(NULL, val, NULL, shsplit);
if (!list || !firstnode(list))
val = dupstring("");

28
Src/zsh.h
View file

@ -1823,6 +1823,34 @@ struct histent {
#define HFILE_NO_REWRITE 0x0020
#define HFILE_USE_OPTIONS 0x8000
/*
* Flags argument to bufferwords() used
* also by lexflags variable.
*/
/*
* Kick the lexer into special string-analysis
* mode without parsing. Any bit set in
* the flags has this effect, but this
* has otherwise all the default effects.
*/
#define LEXFLAGS_ACTIVE 0x0001
/*
* Parse comments and treat each comment as a single string
*/
#define LEXFLAGS_COMMENTS_KEEP 0x0002
/*
* Parse comments and strip them.
*/
#define LEXFLAGS_COMMENTS_STRIP 0x0004
/*
* Either of the above
*/
#define LEXFLAGS_COMMENTS (LEXFLAGS_COMMENTS_KEEP|LEXFLAGS_COMMENTS_STRIP)
/*
* Treat newlines as whitespace
*/
#define LEXFLAGS_NEWLINE 0x0008
/******************************************/
/* Definitions for programable completion */
/******************************************/

11
Test/D04parameter.ztst
View file

@ -462,6 +462,17 @@
>with
>comment
line=$'echo one\necho two # with a comment\necho three'
print -l ${(z+nc+)line}
0:Treating zplit newlines as ordinary whitespace
>echo
>one
>echo
>two
># with a comment
>echo
>three
psvar=(dog)
setopt promptsubst
foo='It shouldn'\''t $(happen) to a %1v.'