45730: _arguments: Add the -0 flag, which makes $opt_args be populated sanely.

Also, write/extend docstrings for sepjoin() and zjoin().

ChangeLog

@@ -1,5 +1,10 @@
 2020-05-03  Daniel Shahaf  <d.s@daniel.shahaf.name>
 
+	* 45730: Completion/Base/Utility/_arguments, Doc/Zsh/compsys.yo,
+	Src/Zle/computil.c, Src/utils.c, Test/Y03arguments.ztst:
+	_arguments: Add the -0 flag, which makes $opt_args be populated
+	sanely.
+
 	* 45729: Src/Modules/curses.c, Src/Zle/compcore.c,
 	Src/Zle/computil.c, Src/builtin.c, Src/linklist.c: internal:
 	Add a second parameter to zlinklist2array(), analogously to

Completion/Base/Utility/_arguments

@@ -7,11 +7,13 @@ local long cmd="$words[1]" descr odescr mesg subopts opt opt2 usecc autod
 local oldcontext="$curcontext" hasopts rawret optarg singopt alwopt
 local setnormarg start rest
 local -a match mbegin mend
+integer opt_args_use_NUL_separators=0
 
 subopts=()
 singopt=()
-while [[ "$1" = -([AMO]*|[CRSWnsw]) ]]; do
+while [[ "$1" = -([AMO]*|[0CRSWnsw]) ]]; do
   case "$1" in
+  -0) opt_args_use_NUL_separators=1; shift ;;
   -C)  usecc=yes; shift ;;
   -O)  subopts=( "${(@P)2}" ); shift 2 ;;
   -O*) subopts=( "${(@P)${1[3,-1]}}" ); shift ;;
@@ -388,7 +390,7 @@ if (( $# )) && comparguments -i "$autod" "$singopt[@]" "$@"; then
             if [[ "$action" = -\>* ]]; then
 	      action="${${action[3,-1]##[ 	]#}%%[ 	]#}"
 	      if (( ! $state[(I)$action] )); then
-                comparguments -W line opt_args
+                comparguments -W line opt_args $opt_args_use_NUL_separators
                 state+=( "$action" )
                 state_descr+=( "$descr" )
 	        if [[ -n "$usecc" ]]; then
@@ -406,7 +408,7 @@ if (( $# )) && comparguments -i "$autod" "$singopt[@]" "$@"; then
                 local=yes
               fi
 
-              comparguments -W line opt_args
+              comparguments -W line opt_args $opt_args_use_NUL_separators
 
               if [[ "$action" = \ # ]]; then
 

Doc/Zsh/compsys.yo

@@ -3740,6 +3740,12 @@ The default var(matchspec) allows partial word completion after `tt(_)' and
 var(matchspec) is:
 example(tt(r:|[_-]=* r:|=*))
 )
+item(tt(-0))(
+When populating values of the `tt(opt_args)' associative array, don't
+backslash-escape colons and backslashes and use NUL rather than colon for
+joining multiple values. This option is described in more detail below, under
+the heading em(var(spec)s: actions).
+)
 enditem()
 
 em(var(spec)s: overview)
@@ -3912,6 +3918,7 @@ specific contexts: on the first call `tt(_arguments $global_options)' is
 used, and on subsequent calls `tt(_arguments !$^global_options)'.
 
 em(var(spec)s: actions)
+COMMENT(If you change this section title, change the references to it in running text.)
 
 In each of the forms above the var(action) determines how
 completions should be generated.  Except for the `tt(->)var(string)'
@@ -4033,9 +4040,21 @@ the normal arguments from the command line, i.e. the words from the
 command line after the command name excluding all options and their
 arguments.  Options are stored in the associative array
 `tt(opt_args)' with option names as keys and their arguments as
-the values.  For options that have more than one argument these are
-given as one string, separated by colons.  All colons and backslashes
-in the original arguments are preceded with backslashes.
+the values.  By default, all colons and backslashes in the value are escaped
+with backslashes, and if an option has multiple arguments (for example, when
+using an var(optspec) of the form `tt(*)var(optspec)'), they are joined with
+(unescaped) colons.  However, if the tt(-0) option was passed, no backslash
+escaping is performed, and multiple values are joined with NUL bytes.  For
+example, after `tt(zsh -o foo:foo -o bar:bar -o <TAB>)', the contents of
+`tt(opt_args)' would be
+
+example(typeset -A opt_args=( [-o]='foo\:foo:bar\:bar:' ))
+
+by default, and
+
+example(typeset -A opt_args=( [-o]=$'foo:foo\x00bar:bar\x00' ))
+
+if tt(_arguments) had been called with the tt(-0) option.
 
 The parameter `tt(context)' is set when returning to the calling function
 to perform an action of the form `tt(->)var(string)'.  It is set to an

Src/Zle/computil.c

@@ -2375,6 +2375,23 @@ ca_parse_line(Cadef d, Cadef all, int multi, int first)
     return 0;
 }
 
+/* Build a NUL-separated from a list.
+ *
+ * This is only used to populate values of $opt_args.
+ */
+
+static char *
+ca_nullist(LinkList l)
+{
+    if (l) {
+	char **array = zlinklist2array(l, 0 /* don't dup elements */);
+	char *ret = zjoin(array, '\0', 0 /* permanent allocation */);
+	free(array); /* the elements are owned by the list */
+	return ret;
+    } else
+	return ztrdup("");
+}
+
 /* Build a colon-list from a list.
  *
  * This is only used to populate values of $opt_args.
@@ -2563,7 +2580,7 @@ bin_comparguments(char *nam, char **args, UNUSED(Options ops), UNUSED(int func))
     case 's': min = 1; max =  1; break;
     case 'M': min = 1; max =  1; break;
     case 'a': min = 0; max =  0; break;
-    case 'W': min = 2; max =  2; break;
+    case 'W': min = 3; max =  3; break;
     case 'n': min = 1; max =  1; break;
     default:
 	zwarnnam(nam, "invalid option: %s", args[0]);
@@ -2795,11 +2812,20 @@ bin_comparguments(char *nam, char **args, UNUSED(Options ops), UNUSED(int func))
 		return 0;
 	return 1;
     case 'W':
-        /* This gets two parameter names as arguments.  The first is set to
-         * the current word sans any option prefixes handled by comparguments.
+        /* This gets two parameter names and one integer as arguments.
+         *
+         * The first parameter is set to the current word sans any option
+         * prefixes handled by comparguments.
+         *
          * The second parameter is set to an array containing the options on
          * the line and their arguments.  I.e. the stuff _arguments returns
-         * to its caller in the `line' and `opt_args' parameters. */
+         * to its caller in the `line' and `opt_args' parameters.
+         *
+         * The integer is one if the second parameter (which is just $opt_args,
+         * you know) should encode multiple values by joining them with NULs
+         * and zero if it should encode multiple values by joining them with
+         * colons after backslash-escaping colons and backslashes.
+         */
 	{
 	    Castate s;
 	    char **ret, **p;
@@ -2807,6 +2833,7 @@ bin_comparguments(char *nam, char **args, UNUSED(Options ops), UNUSED(int func))
 	    LinkList *a;
 	    Caopt o;
 	    int num;
+	    int opt_args_use_NUL_separators = (args[3][0] != '0');
 
 	    for (num = 0, s = lstate; s; s = s->snext)
 		num += countlinknodes(s->args);
@@ -2832,7 +2859,10 @@ bin_comparguments(char *nam, char **args, UNUSED(Options ops), UNUSED(int func))
 		    if (*a) {
 			*p++ = (o->gsname ? tricat(o->gsname, o->name, "") :
 				ztrdup(o->name));
-			*p++ = ca_colonlist(*a);
+			if (opt_args_use_NUL_separators)
+			    *p++ = ca_nullist(*a);
+			else
+			    *p++ = ca_colonlist(*a);
 		    }
 	    *p = NULL;
 

Src/utils.c

@@ -3596,6 +3596,17 @@ strftimehandling:
     return buf - origbuf;
 }
 
+/*
+ * Return a string consisting of the elements of 'arr' joined by the character
+ * 'delim', which will be metafied if necessary.  The string will be allocated
+ * on the heap iff 'heap'.
+ *
+ * Comparable to:
+ *
+ *     char metafied_delim[] = { Meta, delim ^ 32, '\0' };
+ *     sepjoin(arr, metafied_delim, heap)
+ */
+
 /**/
 mod_export char *
 zjoin(char **arr, int delim, int heap)
@@ -3894,10 +3905,12 @@ wordcount(char *s, char *sep, int mul)
 
 /*
  * 's' is a NULL-terminated array of strings.
- * 'sep' is a string.
+ * 'sep' is a string, or NULL to split on ${IFS[1]}.
  *
  * Return a string consisting of the elements of 's' joined by 'sep',
  * allocated on the heap iff 'heap'.
+ *
+ * See also zjoin().
  */
 
 /**/

Test/Y03arguments.ztst

@@ -231,9 +231,15 @@
 
  tst_arguments '-a:one: :two' ':descr:{compadd -Q - $opt_args[-a]}'
  comptest $'tst -a 1:x \\2 \t'
-0:opt_args with multiple arguments and quoting of colons and backslashes
+0:opt_args with multiple arguments and quoting of colons and backslashes, part #1: default behaviour
 >line: {tst -a 1:x \2 1\:x:\\2 }{}
 
+ # Same as previous test, except with -0 and (qqqq) added
+ tst_arguments -0 : '-a:one: :two' ':descr:{compadd -Q - ${(qqqq)opt_args[-a]}}'
+ comptest $'tst -a 1:x \\2 \t'
+0:opt_args with multiple arguments and quoting of colons and backslashes, part #2: NUL escaping
+>line: {tst -a 1:x \2 $'1:x\0\\2' }{}
+
  tst_arguments -a -b
  comptest $'tst  rest -\t\C-w\eb\C-b-\t'
 0:option completion with rest arguments on the line but not in the specs